agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v5 1/7] Row pattern recognition patch for raw parser.
8675+ messages / 2 participants
[nested] [flat]

* [PATCH v5 1/7] Row pattern recognition patch for raw parser.
@ 2023-09-02 06:32 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Tatsuo Ishii @ 2023-09-02 06:32 UTC (permalink / raw)

---
 src/backend/parser/gram.y       | 216 +++++++++++++++++++++++++++++---
 src/include/nodes/parsenodes.h  |  56 +++++++++
 src/include/parser/kwlist.h     |   8 ++
 src/include/parser/parse_node.h |   1 +
 4 files changed, 267 insertions(+), 14 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..70409cdc9a 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	DefElem	   *defelt;
 	SortBy	   *sortby;
 	WindowDef  *windef;
+	RPCommonSyntax	*rpcom;
+	RPSubsetItem	*rpsubset;
 	JoinExpr   *jexpr;
 	IndexElem  *ielem;
 	StatsElem  *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 				TriggerTransitions TriggerReferencing
 				vacuum_relation_list opt_vacuum_relation_list
 				drop_option_list pub_obj_list
-
-%type <node>	opt_routine_body
+				row_pattern_measure_list row_pattern_definition_list
+				opt_row_pattern_subset_clause
+				row_pattern_subset_list row_pattern_subset_rhs
+				row_pattern
+%type <rpsubset>	 row_pattern_subset_item
+%type <node>	opt_routine_body row_pattern_term
 %type <groupclause> group_clause
 %type <list>	group_by_list
 %type <node>	group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 %type <range>	relation_expr_opt_alias
 %type <node>	tablesample_clause opt_repeatable_clause
 %type <target>	target_el set_target insert_column_item
+				row_pattern_measure_item row_pattern_definition
 
 %type <str>		generic_option_name
 %type <node>	generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 %type <list>	window_clause window_definition_list opt_partition_clause
 %type <windef>	window_definition over_clause window_specification
 				opt_frame_clause frame_extent frame_bound
+%type <rpcom>	opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean>	opt_row_pattern_initial_or_seek
+%type <list>	opt_row_pattern_measures
 %type <ival>	opt_window_exclusion_clause
 %type <str>		opt_existing_window_name
 %type <boolean> opt_if_not_exists
@@ -659,7 +669,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 				json_object_constructor_null_clause_opt
 				json_array_constructor_null_clause_opt
 
-
 /*
  * Non-keyword token types.  These are hard-wired into the "flex" lexer.
  * They must be listed first so that their numeric codes do not depend on
@@ -702,7 +711,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
 
 	DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
-	DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+	DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
 	DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
 	DOUBLE_P DROP
 
@@ -718,7 +727,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -731,7 +740,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
 	LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
 
-	MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+	MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
 	MINUTE_P MINVALUE MODE MONTH_P MOVE
 
 	NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +752,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	ORDER ORDINALITY OTHERS OUT_P OUTER_P
 	OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
 
-	PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
-	PLACING PLANS POLICY
+	PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+	PATTERN_P PERMUTE PLACING PLANS POLICY
 	POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
 	PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
 
@@ -755,12 +764,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
 	ROUTINE ROUTINES ROW ROWS RULE
 
-	SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+	SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
 	SEQUENCE SEQUENCES
+
 	SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
 	SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
 	START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
-	SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+	SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
 
 	TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
 	TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +863,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
  */
 %nonassoc	UNBOUNDED		/* ideally would have same precedence as IDENT */
 %nonassoc	IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc	MEASURES AFTER INITIAL SEEK PATTERN_P
 %left		Op OPERATOR		/* multi-character ops and user-defined operators */
 %left		'+' '-'
 %left		'*' '/' '%'
@@ -15857,7 +15868,8 @@ over_clause: OVER window_specification
 		;
 
 window_specification: '(' opt_existing_window_name opt_partition_clause
-						opt_sort_clause opt_frame_clause ')'
+						opt_sort_clause opt_row_pattern_measures opt_frame_clause
+						opt_row_pattern_common_syntax ')'
 				{
 					WindowDef  *n = makeNode(WindowDef);
 
@@ -15865,10 +15877,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
 					n->refname = $2;
 					n->partitionClause = $3;
 					n->orderClause = $4;
+					n->rowPatternMeasures = $5;
 					/* copy relevant fields of opt_frame_clause */
-					n->frameOptions = $5->frameOptions;
-					n->startOffset = $5->startOffset;
-					n->endOffset = $5->endOffset;
+					n->frameOptions = $6->frameOptions;
+					n->startOffset = $6->startOffset;
+					n->endOffset = $6->endOffset;
+					n->rpCommonSyntax = $7;
 					n->location = @1;
 					$$ = n;
 				}
@@ -15892,6 +15906,31 @@ opt_partition_clause: PARTITION BY expr_list		{ $$ = $3; }
 			| /*EMPTY*/								{ $$ = NIL; }
 		;
 
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list	{ $$ = $2; }
+			| /*EMPTY*/								{ $$ = NIL; }
+		;
+
+row_pattern_measure_list:
+			row_pattern_measure_item
+					{ $$ = list_make1($1); }
+			| row_pattern_measure_list ',' row_pattern_measure_item
+					{ $$ = lappend($1, $3); }
+		;
+
+row_pattern_measure_item:
+			a_expr AS ColLabel
+				{
+					$$ = makeNode(ResTarget);
+					$$->name = $3;
+					$$->indirection = NIL;
+					$$->val = (Node *) $1;
+					$$->location = @1;
+				}
+		;
+
 /*
  * For frame clauses, we return a WindowDef, but only some fields are used:
  * frameOptions, startOffset, and endOffset.
@@ -16051,6 +16090,139 @@ opt_window_exclusion_clause:
 			| /*EMPTY*/				{ $$ = 0; }
 		;
 
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+				PATTERN_P '(' row_pattern ')'
+				opt_row_pattern_subset_clause
+				DEFINE row_pattern_definition_list
+			{
+				RPCommonSyntax *n = makeNode(RPCommonSyntax);
+				n->rpSkipTo = $1->rpSkipTo;
+				n->rpSkipVariable = $1->rpSkipVariable;
+				n->initial = $2;
+				n->rpPatterns = $5;
+				n->rpSubsetClause = $7;
+				n->rpDefs = $9;
+				$$ = n;
+			}
+			| /*EMPTY*/		{ $$ = NULL; }
+	;
+
+opt_row_pattern_skip_to:
+			AFTER MATCH SKIP TO NEXT ROW
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					n->rpSkipTo = ST_NEXT_ROW;
+					n->rpSkipVariable = NULL;
+					$$ = n;
+			}
+			| AFTER MATCH SKIP PAST LAST_P ROW
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					n->rpSkipTo = ST_PAST_LAST_ROW;
+					n->rpSkipVariable = NULL;
+					$$ = n;
+				}
+/*
+			| AFTER MATCH SKIP TO FIRST_P ColId		%prec FIRST_P
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					n->rpSkipTo = ST_FIRST_VARIABLE;
+					n->rpSkipVariable = $6;
+					$$ = n;
+				}
+			| AFTER MATCH SKIP TO LAST_P ColId		%prec LAST_P
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					n->rpSkipTo = ST_LAST_VARIABLE;
+					n->rpSkipVariable = $6;
+					$$ = n;
+				}
+ * Shift/reduce
+			| AFTER MATCH SKIP TO ColId
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					n->rpSkipTo = ST_VARIABLE;
+					n->rpSkipVariable = $5;
+					$$ = n;
+				}
+*/
+			| /*EMPTY*/
+				{
+					RPCommonSyntax *n = makeNode(RPCommonSyntax);
+					/* temporary set default to ST_NEXT_ROW */
+					n->rpSkipTo = ST_PAST_LAST_ROW;
+					n->rpSkipVariable = NULL;
+					$$ = n;
+				}
+	;
+
+opt_row_pattern_initial_or_seek:
+			INITIAL			{ $$ = true; }
+			| SEEK
+				{
+					ereport(ERROR,
+							(errcode(ERRCODE_SYNTAX_ERROR),
+							 errmsg("SEEK is not supported"),
+							 errhint("Use INITIAL."),
+							 parser_errposition(@1)));
+				}
+			| /*EMPTY*/		{ $$ = true; }
+		;
+
+row_pattern:
+			row_pattern_term							{ $$ = list_make1($1); }
+			| row_pattern row_pattern_term				{ $$ = lappend($1, $2); }
+		;
+
+row_pattern_term:
+			ColId	{ $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+			| ColId '*'	{ $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+			| ColId '+'	{ $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+			| ColId '?'	{ $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+		;
+
+opt_row_pattern_subset_clause:
+			SUBSET row_pattern_subset_list	{ $$ = $2; }
+			| /*EMPTY*/												{ $$ = NIL; }
+		;
+
+row_pattern_subset_list:
+			row_pattern_subset_item									{ $$ = list_make1($1); }
+			| row_pattern_subset_list ',' row_pattern_subset_item	{ $$ = lappend($1, $3); }
+			| /*EMPTY*/												{ $$ = NIL; }
+		;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+			{
+				RPSubsetItem *n = makeNode(RPSubsetItem);
+				n->name = $1;
+				n->rhsVariable = $4;
+				$$ = n;
+			}
+		;
+
+row_pattern_subset_rhs:
+			ColId								{ $$ = list_make1(makeStringConst($1, @1)); }
+			| row_pattern_subset_rhs ',' ColId	{ $$ = lappend($1, makeStringConst($3, @1)); }
+			| /*EMPTY*/							{ $$ = NIL; }
+		;
+
+row_pattern_definition_list:
+			row_pattern_definition										{ $$ = list_make1($1); }
+			| row_pattern_definition_list ',' row_pattern_definition	{ $$ = lappend($1, $3); }
+		;
+
+row_pattern_definition:
+			ColId AS a_expr
+				{
+					$$ = makeNode(ResTarget);
+					$$->name = $1;
+					$$->indirection = NIL;
+					$$->val = (Node *) $3;
+					$$->location = @1;
+				}
+		;
 
 /*
  * Supporting nonterminals for expressions.
@@ -17146,6 +17318,7 @@ unreserved_keyword:
 			| INDEXES
 			| INHERIT
 			| INHERITS
+			| INITIAL
 			| INLINE_P
 			| INPUT_P
 			| INSENSITIVE
@@ -17173,6 +17346,7 @@ unreserved_keyword:
 			| MATCHED
 			| MATERIALIZED
 			| MAXVALUE
+			| MEASURES
 			| MERGE
 			| METHOD
 			| MINUTE_P
@@ -17215,6 +17389,9 @@ unreserved_keyword:
 			| PARTITION
 			| PASSING
 			| PASSWORD
+			| PAST
+			| PATTERN_P
+			| PERMUTE
 			| PLANS
 			| POLICY
 			| PRECEDING
@@ -17265,6 +17442,7 @@ unreserved_keyword:
 			| SEARCH
 			| SECOND_P
 			| SECURITY
+			| SEEK
 			| SEQUENCE
 			| SEQUENCES
 			| SERIALIZABLE
@@ -17290,6 +17468,7 @@ unreserved_keyword:
 			| STRICT_P
 			| STRIP_P
 			| SUBSCRIPTION
+			| SUBSET
 			| SUPPORT
 			| SYSID
 			| SYSTEM_P
@@ -17477,6 +17656,7 @@ reserved_keyword:
 			| CURRENT_USER
 			| DEFAULT
 			| DEFERRABLE
+			| DEFINE
 			| DESC
 			| DISTINCT
 			| DO
@@ -17639,6 +17819,7 @@ bare_label_keyword:
 			| DEFAULTS
 			| DEFERRABLE
 			| DEFERRED
+			| DEFINE
 			| DEFINER
 			| DELETE_P
 			| DELIMITER
@@ -17714,6 +17895,7 @@ bare_label_keyword:
 			| INDEXES
 			| INHERIT
 			| INHERITS
+			| INITIAL
 			| INITIALLY
 			| INLINE_P
 			| INNER_P
@@ -17763,6 +17945,7 @@ bare_label_keyword:
 			| MATCHED
 			| MATERIALIZED
 			| MAXVALUE
+			| MEASURES
 			| MERGE
 			| METHOD
 			| MINVALUE
@@ -17816,6 +17999,9 @@ bare_label_keyword:
 			| PARTITION
 			| PASSING
 			| PASSWORD
+			| PAST
+			| PATTERN_P
+			| PERMUTE
 			| PLACING
 			| PLANS
 			| POLICY
@@ -17872,6 +18058,7 @@ bare_label_keyword:
 			| SCROLL
 			| SEARCH
 			| SECURITY
+			| SEEK
 			| SELECT
 			| SEQUENCE
 			| SEQUENCES
@@ -17903,6 +18090,7 @@ bare_label_keyword:
 			| STRICT_P
 			| STRIP_P
 			| SUBSCRIPTION
+			| SUBSET
 			| SUBSTRING
 			| SUPPORT
 			| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fef4c714b8..4cc1fb417b 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
 	int			location;		/* operator location, or -1 if none/unknown */
 } SortBy;
 
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+	ST_NONE,					/* AFTER MATCH omitted */
+	ST_NEXT_ROW,				/* SKIP TO NEXT ROW */
+	ST_PAST_LAST_ROW,			/* SKIP TO PAST LAST ROW */
+	ST_FIRST_VARIABLE,			/* SKIP TO FIRST variable name */
+	ST_LAST_VARIABLE,			/* SKIP TO LAST variable name */
+	ST_VARIABLE					/* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+	NodeTag		type;
+	char	   *name;			/* Row Pattern SUBSET clause variable name */
+	List	   *rhsVariable;	/* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+	NodeTag		type;
+	RPSkipTo	rpSkipTo;		/* Row Pattern AFTER MATCH SKIP type */
+	char	   *rpSkipVariable;	/* Row Pattern Skip To variable name, if any */
+	bool		initial;		/* true if <row pattern initial or seek> is initial */
+	List	   *rpPatterns;		/* PATTERN variables (list of A_Expr) */
+	List	   *rpSubsetClause;	/* row pattern subset clause (list of RPSubsetItem), if any */
+	List	   *rpDefs;			/* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
 /*
  * WindowDef - raw representation of WINDOW and OVER clauses
  *
@@ -562,6 +600,8 @@ typedef struct WindowDef
 	char	   *refname;		/* referenced window name, if any */
 	List	   *partitionClause;	/* PARTITION BY expression list */
 	List	   *orderClause;	/* ORDER BY (list of SortBy) */
+	List	   *rowPatternMeasures;	/* row pattern measures (list of ResTarget) */
+	RPCommonSyntax *rpCommonSyntax;	/* row pattern common syntax */
 	int			frameOptions;	/* frame_clause options, see below */
 	Node	   *startOffset;	/* expression for starting bound, if any */
 	Node	   *endOffset;		/* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
  * the orderClause might or might not be copied (see copiedOrder); the framing
  * options are never copied, per spec.
  *
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ * 
  * The information relevant for the query jumbling is the partition clause
  * type and its bounds.
  */
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
 	Index		winref;			/* ID referenced by window functions */
 	/* did we copy orderClause from refname? */
 	bool		copiedOrder pg_node_attr(query_jumble_ignore);
+	/* Row Pattern AFTER MACH SKIP clause */
+	RPSkipTo	rpSkipTo;		/* Row Pattern Skip To type */
+	bool		initial;		/* true if <row pattern initial or seek> is initial */
+	/* Row Pattern DEFINE clause (list of TargetEntry) */
+	List		*defineClause;
+	/* Row Pattern DEFINE variable initial names (list of String) */
+	List		*defineInitial;
+	/* Row Pattern PATTERN variable name (list of String) */
+	List		*patternVariable;
+	/* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+	List		*patternRegexp;
 } WindowClause;
 
 /*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
 PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
 PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
 	EXPR_KIND_WINDOW_FRAME_RANGE,	/* window frame clause with RANGE */
 	EXPR_KIND_WINDOW_FRAME_ROWS,	/* window frame clause with ROWS */
 	EXPR_KIND_WINDOW_FRAME_GROUPS,	/* window frame clause with GROUPS */
+	EXPR_KIND_RPR_DEFINE,		/* DEFINE */
 	EXPR_KIND_SELECT_TARGET,	/* SELECT target list item */
 	EXPR_KIND_INSERT_TARGET,	/* INSERT target list item */
 	EXPR_KIND_UPDATE_SOURCE,	/* UPDATE assignment source item */
-- 
2.25.1


----Next_Part(Sat_Sep__2_15_52_35_2023_273)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v5-0002-Row-pattern-recognition-patch-parse-analysis.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..97ea882cf71 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index 00b21ede481..c25dbeadff3 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -233,6 +234,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0f6b5df322c..16009fa75a4 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1470,7 +1471,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid		dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index fe6bfeba25e..0d415f5f8f2 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1776,9 +1777,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid		dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin
+		 * is not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..49d1bf4bfac 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+					continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--=-=-=--





^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v50 7/8] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--brnevsqjnuzpyok4
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v50-0008-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v52 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v52-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread

* [PATCH v51 08/10] Introduce an option to make logical replication database specific.
@ 2026-04-03 10:34 Antonin Houska <[email protected]>
  0 siblings, 0 replies; 8675+ messages in thread

From: Antonin Houska @ 2026-04-03 10:34 UTC (permalink / raw)

By default, the logical decoding assumes access to shared catalogs, so the
snapshot builder needs to consider cluster-wide XIDs during startup. That in
turn means that, if any transaction is already running (and has XID assigned),
the snapshot builder needs to wait for its completion, as it does not know if
that transaction performed catalog changes earlier.

Possible problem with this concept is that if REPACK (CONCURRENTLY) is running
in some database, backends running the same command in other databases get
stuck until the first one has committed. Thus only as single backend in the
cluster can run REPACK (CONCURRENTLY) at any time. Likewise, REPACK
(CONCURRENTLY) can block walsenders starting on behalf of subscriptions
throughout the cluster.

This patch introduces the possibility for a backend to declare that its output
plugin does not use shared catalogs (i.e. catalogs that can be changed by
transactions running in other databases in the cluster). In that case, no
snapshot the backend will use during the decoding needs to contain information
about transactions running in other databases. Thus the snapshot builder only
needs to wait for completion of transactions in the current database.

Currently we only use this option in the REPACK background worker. It could
possibly be used in walsender involved in logical replication too, however
that would need thorough analysis of its output plugin.

The patch bumps WAL version number, due to a new field in xl_running_xacts.
---
 contrib/pg_visibility/pg_visibility.c       |  4 ++--
 src/backend/access/index/genam.c            | 18 ++++++++++++++++
 src/backend/access/rmgrdesc/standbydesc.c   |  2 ++
 src/backend/access/transam/xlog.c           |  2 +-
 src/backend/access/transam/xlogfuncs.c      |  2 +-
 src/backend/commands/repack_worker.c        |  8 +++++++
 src/backend/postmaster/bgwriter.c           |  2 +-
 src/backend/replication/logical/snapbuild.c | 10 ++++++++-
 src/backend/replication/slot.c              | 12 +++++++++--
 src/backend/storage/ipc/procarray.c         | 23 ++++++++++++++++++++-
 src/backend/storage/ipc/standby.c           | 14 +++++++++++--
 src/include/access/genam.h                  |  8 +++++++
 src/include/access/xlog_internal.h          |  2 +-
 src/include/storage/procarray.h             |  2 +-
 src/include/storage/standby.h               |  3 ++-
 src/include/storage/standbydefs.h           |  1 +
 16 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c
index dfab0b64cf5..d564bd2a00c 100644
--- a/contrib/pg_visibility/pg_visibility.c
+++ b/contrib/pg_visibility/pg_visibility.c
@@ -621,7 +621,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 	else if (rel == NULL || rel->rd_rel->relisshared)
 	{
 		/* Shared relation: take into account all running xids */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestRunningXid;
@@ -632,7 +632,7 @@ GetStrictOldestNonRemovableTransactionId(Relation rel)
 		 * Normal relation: take into account xids running within the current
 		 * database
 		 */
-		runningTransactions = GetRunningTransactionData();
+		runningTransactions = GetRunningTransactionData(InvalidOid);
 		LWLockRelease(ProcArrayLock);
 		LWLockRelease(XidGenLock);
 		return runningTransactions->oldestDatabaseRunningXid;
diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c
index 1408989c568..df092dc999a 100644
--- a/src/backend/access/index/genam.c
+++ b/src/backend/access/index/genam.c
@@ -37,6 +37,14 @@
 #include "utils/ruleutils.h"
 #include "utils/snapmgr.h"
 
+/*
+ * If a backend is going to do logical decoding and if the output plugin does
+ * not need to access shared catalogs, setting this variable to false can make
+ * the decoding startup faster. In particular, the backend will not need to
+ * wait for completion of already running transactions in other databases.
+ */
+bool		accessSharedCatalogsInDecoding = true;
+
 
 /* ----------------------------------------------------------------
  *		general access method routines
@@ -394,6 +402,16 @@ systable_beginscan(Relation heapRelation,
 	SysScanDesc sysscan;
 	Relation	irel;
 
+	/*
+	 * If this backend promised that it won't access shared catalogs during
+	 * logical decoding, this seems to be the right place to check.
+	 *
+	 * XXX Should this be ereport(ERROR) ?
+	 */
+	Assert(!HistoricSnapshotActive() ||
+		   accessSharedCatalogsInDecoding ||
+		   !heapRelation->rd_rel->relisshared);
+
 	if (indexOK &&
 		!IgnoreSystemIndexes &&
 		!ReindexIsProcessingIndex(indexId))
diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c
index 0a291354ae2..685d1bdb024 100644
--- a/src/backend/access/rmgrdesc/standbydesc.c
+++ b/src/backend/access/rmgrdesc/standbydesc.c
@@ -41,6 +41,8 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec)
 		for (i = 0; i < xlrec->subxcnt; i++)
 			appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]);
 	}
+
+	appendStringInfo(buf, "; dbid: %u", xlrec->dbid);
 }
 
 void
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2c1c6f88b74..8da8eff93e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7336,7 +7336,7 @@ CreateCheckPoint(int flags)
 	 * recovery we don't need to write running xact data.
 	 */
 	if (!shutdown && XLogStandbyInfoActive())
-		LogStandbySnapshot();
+		LogStandbySnapshot(InvalidOid);
 
 	START_CRIT_SECTION();
 
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index 65bbaeda59c..0f5979691e6 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -245,7 +245,7 @@ pg_log_standby_snapshot(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("pg_log_standby_snapshot() can only be used if \"wal_level\" >= \"replica\"")));
 
-	recptr = LogStandbySnapshot();
+	recptr = LogStandbySnapshot(InvalidOid);
 
 	/*
 	 * As a convenience, return the WAL location of the last inserted record
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index c85166ba849..ff34e246469 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -16,6 +16,7 @@
  */
 #include "postgres.h"
 
+#include "access/genam.h"
 #include "access/table.h"
 #include "access/xlog_internal.h"
 #include "access/xlogutils.h"
@@ -237,6 +238,13 @@ repack_setup_logical_decoding(Oid relid)
 
 	EnsureLogicalDecodingEnabled();
 
+	/*
+	 * By declaring that our output plugin does not need shared catalogs, we
+	 * avoid waiting for completion of transactions running in other databases
+	 * than the one we're connected to.
+	 */
+	accessSharedCatalogsInDecoding = false;
+
 	/*
 	 * Neither prepare_write nor do_write callback nor update_progress is
 	 * useful for us.
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 1d8947774a9..a30de4262eb 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -289,7 +289,7 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
 			if (now >= timeout &&
 				last_snapshot_lsn <= GetLastImportantRecPtr())
 			{
-				last_snapshot_lsn = LogStandbySnapshot();
+				last_snapshot_lsn = LogStandbySnapshot(InvalidOid);
 				last_snapshot_ts = now;
 			}
 		}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index b4269a3b102..2e3926e1d13 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include "access/genam.h"
 #include "access/heapam_xlog.h"
 #include "access/transam.h"
 #include "access/xact.h"
@@ -1465,7 +1466,14 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff)
 	 */
 	if (!RecoveryInProgress())
 	{
-		LogStandbySnapshot();
+		Oid			dbid;
+
+		/*
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		LogStandbySnapshot(dbid);
 	}
 }
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index a9092fc2382..d553bd5dbff 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/genam.h"
 #include "access/transam.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
@@ -1760,9 +1761,16 @@ ReplicationSlotReserveWal(void)
 	if (!RecoveryInProgress() && SlotIsLogical(slot))
 	{
 		XLogRecPtr	flushptr;
+		Oid			dbid;
 
-		/* make sure we have enough information to start */
-		flushptr = LogStandbySnapshot();
+		/*
+		 * Make sure we have enough information to start.
+		 *
+		 * Only consider transactions of the current database if our plugin is
+		 * not supposed to access shared catalogs.
+		 */
+		dbid = accessSharedCatalogsInDecoding ? InvalidOid : MyDatabaseId;
+		flushptr = LogStandbySnapshot(dbid);
 
 		/* and make sure it's fsynced to disk */
 		XLogFlush(flushptr);
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index cc207cb56e3..92ed34128e5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -2631,9 +2631,11 @@ ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc)
  *
  * Note that if any transaction has overflowed its cached subtransactions
  * then there is no real need include any subtransactions.
+ *
+ * If 'dbid' is valid, only gather transactions running in that database.
  */
 RunningTransactions
-GetRunningTransactionData(void)
+GetRunningTransactionData(Oid dbid)
 {
 	/* result workspace */
 	static RunningTransactionsData CurrentRunningXactsData;
@@ -2708,6 +2710,18 @@ GetRunningTransactionData(void)
 		if (!TransactionIdIsValid(xid))
 			continue;
 
+		/*
+		 * Filter by database OID if requested.
+		 */
+		if (OidIsValid(dbid))
+		{
+			int			pgprocno = arrayP->pgprocnos[index];
+			PGPROC	   *proc = &allProcs[pgprocno];
+
+			if (proc->databaseId != dbid)
+				continue;
+		}
+
 		/*
 		 * Be careful not to exclude any xids before calculating the values of
 		 * oldestRunningXid and suboverflowed, since these are used to clean
@@ -2758,6 +2772,12 @@ GetRunningTransactionData(void)
 			PGPROC	   *proc = &allProcs[pgprocno];
 			int			nsubxids;
 
+			/*
+			 * Filter by database OID if requested.
+			 */
+			if (OidIsValid(dbid) && proc->databaseId != dbid)
+				continue;
+
 			/*
 			 * Save subtransaction XIDs. Other backends can't add or remove
 			 * entries while we're holding XidGenLock.
@@ -2791,6 +2811,7 @@ GetRunningTransactionData(void)
 	 * increases if slots do.
 	 */
 
+	CurrentRunningXacts->dbid = dbid;
 	CurrentRunningXacts->xcnt = count - subcount;
 	CurrentRunningXacts->subxcnt = subcount;
 	CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY;
diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index de9092fdf5b..c653ea742bc 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -1188,6 +1188,14 @@ standby_redo(XLogReaderState *record)
 		xl_running_xacts *xlrec = (xl_running_xacts *) XLogRecGetData(record);
 		RunningTransactionsData running;
 
+		/*
+		 * Records issued for specific database are not suitable for physical
+		 * replication because that affects the whole cluster. In particular,
+		 * the list of XID is probably incomplete here.
+		 */
+		if (OidIsValid(xlrec->dbid))
+			return;
+
 		running.xcnt = xlrec->xcnt;
 		running.subxcnt = xlrec->subxcnt;
 		running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY;
@@ -1277,11 +1285,12 @@ standby_redo(XLogReaderState *record)
  * as there's no independent knob to just enable logical decoding. For
  * details of how this is used, check snapbuild.c's introductory comment.
  *
+ * If 'dbid' is valid, only gather transactions running in that database.
  *
  * Returns the RecPtr of the last inserted record.
  */
 XLogRecPtr
-LogStandbySnapshot(void)
+LogStandbySnapshot(Oid dbid)
 {
 	XLogRecPtr	recptr;
 	RunningTransactions running;
@@ -1314,7 +1323,7 @@ LogStandbySnapshot(void)
 	 * Log details of all in-progress transactions. This should be the last
 	 * record we write, because standby will open up when it sees this.
 	 */
-	running = GetRunningTransactionData();
+	running = GetRunningTransactionData(dbid);
 
 	/*
 	 * GetRunningTransactionData() acquired ProcArrayLock, we must release it.
@@ -1358,6 +1367,7 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts)
 	xl_running_xacts xlrec;
 	XLogRecPtr	recptr;
 
+	xlrec.dbid = CurrRunningXacts->dbid;
 	xlrec.xcnt = CurrRunningXacts->xcnt;
 	xlrec.subxcnt = CurrRunningXacts->subxcnt;
 	xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY);
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index b69320a7fc8..56c573399ab 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -136,6 +136,14 @@ typedef struct IndexOrderByDistance
 	bool		isnull;
 } IndexOrderByDistance;
 
+/*
+ * Is the backend interested in shared catalogs when performing logical
+ * decoding?
+ *
+ * XXX Is there a better place for this declaration?
+ */
+extern bool accessSharedCatalogsInDecoding;
+
 /*
  * generalized index_ interface routines (in indexam.c)
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 755835d63bf..ae19982d88d 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -31,7 +31,7 @@
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD11E	/* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD11F	/* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index abdf021e66e..377b3060b9f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -49,7 +49,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin,
 										 VirtualTransactionId *sourcevxid);
 extern bool ProcArrayInstallRestoredXmin(TransactionId xmin, PGPROC *proc);
 
-extern RunningTransactions GetRunningTransactionData(void);
+extern RunningTransactions GetRunningTransactionData(Oid dbid);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern TransactionId GetOldestNonRemovableTransactionId(Relation rel);
diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h
index 6a314c693cd..8715c08e94f 100644
--- a/src/include/storage/standby.h
+++ b/src/include/storage/standby.h
@@ -126,6 +126,7 @@ typedef enum
 
 typedef struct RunningTransactionsData
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	subxids_array_status subxid_status;
@@ -143,7 +144,7 @@ typedef RunningTransactionsData *RunningTransactions;
 extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid);
 extern void LogAccessExclusiveLockPrepare(void);
 
-extern XLogRecPtr LogStandbySnapshot(void);
+extern XLogRecPtr LogStandbySnapshot(Oid dbid);
 extern void LogStandbyInvalidations(int nmsgs, SharedInvalidationMessage *msgs,
 									bool relcacheInitFileInval);
 
diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h
index 231d251fd51..e75b7078766 100644
--- a/src/include/storage/standbydefs.h
+++ b/src/include/storage/standbydefs.h
@@ -46,6 +46,7 @@ typedef struct xl_standby_locks
  */
 typedef struct xl_running_xacts
 {
+	Oid			dbid;			/* only track xacts in this database */
 	int			xcnt;			/* # of xact ids in xids[] */
 	int			subxcnt;		/* # of subxact ids in xids[] */
 	bool		subxid_overflow;	/* snapshot overflowed, subxids missing */
-- 
2.47.3


--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
	filename="v51-0009-Reserve-replication-slots-specifically-for-REPAC.patch"



^ permalink  raw  reply  [nested|flat] 8675+ messages in thread