public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 3/3] Aggregate push-down - basic functionality. 55+ messages / 7 participants [nested] [flat]
* [PATCH 3/3] Aggregate push-down - basic functionality. @ 2019-07-17 14:31 Antonin Houska <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: Antonin Houska @ 2019-07-17 14:31 UTC (permalink / raw) --- src/backend/nodes/copyfuncs.c | 21 +- src/backend/nodes/outfuncs.c | 36 ++ src/backend/optimizer/README | 87 +++ src/backend/optimizer/path/allpaths.c | 112 ++++ src/backend/optimizer/path/costsize.c | 17 +- src/backend/optimizer/path/equivclass.c | 134 +++++ src/backend/optimizer/path/joinrels.c | 170 +++++- src/backend/optimizer/plan/initsplan.c | 257 ++++++++ src/backend/optimizer/plan/planagg.c | 2 +- src/backend/optimizer/plan/planmain.c | 23 +- src/backend/optimizer/plan/planner.c | 58 +- src/backend/optimizer/plan/setrefs.c | 33 ++ src/backend/optimizer/prep/prepjointree.c | 1 + src/backend/optimizer/util/pathnode.c | 143 ++++- src/backend/optimizer/util/relnode.c | 916 +++++++++++++++++++++++++++-- src/backend/optimizer/util/tlist.c | 56 ++ src/backend/utils/misc/guc.c | 9 + src/include/nodes/nodes.h | 2 + src/include/nodes/pathnodes.h | 99 ++++ src/include/optimizer/clauses.h | 2 + src/include/optimizer/pathnode.h | 19 +- src/include/optimizer/paths.h | 7 + src/include/optimizer/planmain.h | 4 +- src/include/optimizer/tlist.h | 8 +- src/test/regress/expected/agg_pushdown.out | 217 +++++++ src/test/regress/expected/sysviews.out | 3 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/agg_pushdown.sql | 117 ++++ 28 files changed, 2480 insertions(+), 74 deletions(-) create mode 100644 src/test/regress/expected/agg_pushdown.out create mode 100644 src/test/regress/sql/agg_pushdown.sql diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 6414aded0e..097872f851 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2204,8 +2204,8 @@ _copyOnConflictExpr(const OnConflictExpr *from) /* **************************************************************** * pathnodes.h copy functions * - * We don't support copying RelOptInfo, IndexOptInfo, or Path nodes. - * There are some subsidiary structs that are useful to copy, though. + * We don't support copying RelOptInfo, IndexOptInfo, RelAggInfo or Path + * nodes. There are some subsidiary structs that are useful to copy, though. * **************************************************************** */ @@ -2346,6 +2346,20 @@ _copyPlaceHolderInfo(const PlaceHolderInfo *from) return newnode; } +static GroupedVarInfo * +_copyGroupedVarInfo(const GroupedVarInfo *from) +{ + GroupedVarInfo *newnode = makeNode(GroupedVarInfo); + + COPY_NODE_FIELD(gvexpr); + COPY_NODE_FIELD(agg_partial); + COPY_SCALAR_FIELD(sortgroupref); + COPY_SCALAR_FIELD(gv_eval_at); + COPY_SCALAR_FIELD(derived); + + return newnode; +} + /* **************************************************************** * parsenodes.h copy functions * **************************************************************** @@ -5082,6 +5096,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; + case T_GroupedVarInfo: + retval = _copyGroupedVarInfo(from); + break; /* * VALUE NODES diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 01745ff879..49ee37fc7b 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2190,6 +2190,8 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_BITMAPSET_FIELD(all_baserels); WRITE_BITMAPSET_FIELD(nullable_baserels); WRITE_NODE_FIELD(join_rel_list); + WRITE_NODE_FIELD(grouped_rel_list); + WRITE_NODE_FIELD(agg_info_list); WRITE_INT_FIELD(join_cur_level); WRITE_NODE_FIELD(init_plans); WRITE_NODE_FIELD(cte_plan_ids); @@ -2203,6 +2205,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(append_rel_list); WRITE_NODE_FIELD(rowMarks); WRITE_NODE_FIELD(placeholder_list); + WRITE_NODE_FIELD(grouped_var_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); WRITE_NODE_FIELD(group_pathkeys); @@ -2210,6 +2213,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(distinct_pathkeys); WRITE_NODE_FIELD(sort_pathkeys); WRITE_NODE_FIELD(processed_tlist); + WRITE_INT_FIELD(max_sortgroupref); WRITE_NODE_FIELD(minmax_aggs); WRITE_FLOAT_FIELD(total_table_pages, "%.0f"); WRITE_FLOAT_FIELD(tuple_fraction, "%.4f"); @@ -2429,6 +2433,20 @@ _outParamPathInfo(StringInfo str, const ParamPathInfo *node) } static void +_outRelAggInfo(StringInfo str, const RelAggInfo *node) +{ + WRITE_NODE_TYPE("RELAGGINFO"); + + WRITE_BITMAPSET_FIELD(relids); + WRITE_NODE_FIELD(target); + WRITE_NODE_FIELD(agg_input); + WRITE_FLOAT_FIELD(input_rows, "%.0f"); + WRITE_NODE_FIELD(group_clauses); + WRITE_NODE_FIELD(group_exprs); + WRITE_NODE_FIELD(agg_exprs); +} + +static void _outRestrictInfo(StringInfo str, const RestrictInfo *node) { WRITE_NODE_TYPE("RESTRICTINFO"); @@ -2528,6 +2546,18 @@ _outPlaceHolderInfo(StringInfo str, const PlaceHolderInfo *node) } static void +_outGroupedVarInfo(StringInfo str, const GroupedVarInfo *node) +{ + WRITE_NODE_TYPE("GROUPEDVARINFO"); + + WRITE_NODE_FIELD(gvexpr); + WRITE_NODE_FIELD(agg_partial); + WRITE_UINT_FIELD(sortgroupref); + WRITE_BITMAPSET_FIELD(gv_eval_at); + WRITE_BOOL_FIELD(derived); +} + +static void _outMinMaxAggInfo(StringInfo str, const MinMaxAggInfo *node) { WRITE_NODE_TYPE("MINMAXAGGINFO"); @@ -4084,6 +4114,9 @@ outNode(StringInfo str, const void *obj) case T_ParamPathInfo: _outParamPathInfo(str, obj); break; + case T_RelAggInfo: + _outRelAggInfo(str, obj); + break; case T_RestrictInfo: _outRestrictInfo(str, obj); break; @@ -4102,6 +4135,9 @@ outNode(StringInfo str, const void *obj) case T_PlaceHolderInfo: _outPlaceHolderInfo(str, obj); break; + case T_GroupedVarInfo: + _outGroupedVarInfo(str, obj); + break; case T_MinMaxAggInfo: _outMinMaxAggInfo(str, obj); break; diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README index 89ce373d5e..4633fb8768 100644 --- a/src/backend/optimizer/README +++ b/src/backend/optimizer/README @@ -1127,3 +1127,90 @@ breaking down aggregation or grouping over a partitioned relation into aggregation or grouping over its partitions is called partitionwise aggregation. Especially when the partition keys match the GROUP BY clause, this can be significantly faster than the regular method. + +Aggregate push-down +------------------- + +The obvious way to evaluate aggregates is to evaluate the FROM clause of the +SQL query (this is what query_planner does) and use the resuing paths as the +input of Agg node. However, if the groups are large enough, it may be more +efficient to apply the partial aggregation to the output of base relation +scan, and finalize it when we have all relations of the query joined: + + EXPLAIN + SELECT a.i, avg(b.u) + FROM a JOIN b ON b.j = a.i + GROUP BY a.i; + + Finalize HashAggregate + Group Key: a.i + -> Nested Loop + -> Partial HashAggregate + Group Key: b.j + -> Seq Scan on b + -> Index Only Scan using a_pkey on a + Index Cond: (i = b.j) + +Thus the join above the partial aggregate node receives fewer input rows, and +so the number of outer-to-inner pairs of tuples to be checked can be +significantly lower, which can in turn lead to considerably lower join cost. + +Note that there's often no GROUP BY expression to be used for the partial +aggregation, so we use equivalence classes to derive grouping expression: in +the example above, the grouping key "b.j" was derived from "a.i". + +Furthermore, extra grouping columns can be added to the partial Agg node if a +join clause above that node references a column which is not in the query +GROUP BY clause and which could not be derived using equivalence class. + + EXPLAIN + SELECT a.i, avg(b.u) + FROM a JOIN b ON b.j = a.x + GROUP BY a.i; + + Finalize HashAggregate + Group Key: a.i + -> Hash Join + Hash Cond: (a.x = b.j) + -> Seq Scan on a + -> Hash + -> Partial HashAggregate + Group Key: b.j + -> Seq Scan on b + +Here the partial aggregate uses "b.j" as grouping column although it's not in +the same equivalence class as "a.i". Note that no column of the {a.x, b.j} +equivalence class is used as a key for the final aggregation. + +Besides base relation, the aggregation can also be pushed down to join: + + EXPLAIN + SELECT a.i, avg(b.u + c.v) + FROM a JOIN b ON b.j = a.i + JOIN c ON c.k = a.i + WHERE b.j = c.k GROUP BY a.i; + + Finalize HashAggregate + Group Key: a.i + -> Hash Join + Hash Cond: (b.j = a.i) + -> Partial HashAggregate + Group Key: b.j + -> Hash Join + Hash Cond: (b.j = c.k) + -> Seq Scan on b + -> Hash + -> Seq Scan on c + -> Hash + -> Seq Scan on a + +Whether the Agg node is created out of base relation or out of join, it's +added to a separate RelOptInfo that we call "grouped relation". Grouped +relation can be joined to a non-grouped relation, which results in a grouped +relation too. Join of two grouped relations does not seem to be very useful +and is currently not supported. + +If query_planner produces a grouped relation that contains valid paths, these +are simply added to the UPPERREL_PARTIAL_GROUP_AGG relation. Further +processing of these paths then does not differ from processing of other +partially grouped paths. diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index e9ee32b7f4..e073732ac8 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -60,6 +60,7 @@ typedef struct pushdown_safety_info /* These parameters are set by GUC */ bool enable_geqo = false; /* just in case GUC doesn't set it */ +bool enable_agg_pushdown; int geqo_threshold; int min_parallel_table_scan_size; int min_parallel_index_scan_size; @@ -124,6 +125,9 @@ static void set_result_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); +static void add_grouped_path(PlannerInfo *root, RelOptInfo *rel, + Path *subpath, AggStrategy aggstrategy, + RelAggInfo *agg_info); static RelOptInfo *make_rel_from_joinlist(PlannerInfo *root, List *joinlist); static bool subquery_is_pushdown_safe(Query *subquery, Query *topquery, pushdown_safety_info *safetyInfo); @@ -2731,6 +2735,80 @@ generate_gather_paths(PlannerInfo *root, RelOptInfo *rel, bool override_rows) } /* + * generate_grouping_paths + * Create partially aggregated paths and add them to grouped relation. + * + * "rel_plain" is base or join relation whose paths are not grouped. + */ +void +generate_grouping_paths(PlannerInfo *root, RelOptInfo *rel_grouped, + RelOptInfo *rel_plain, RelAggInfo *agg_info) +{ + ListCell *lc; + + if (IS_DUMMY_REL(rel_plain)) + { + mark_dummy_rel(rel_grouped); + return; + } + + foreach(lc, rel_plain->pathlist) + { + Path *path = (Path *) lfirst(lc); + + /* + * Since the path originates from the non-grouped relation which is + * not aware of the aggregate push-down, we must ensure that it + * provides the correct input for aggregation. + */ + path = (Path *) create_projection_path(root, rel_grouped, path, + agg_info->agg_input); + + /* + * add_grouped_path() will check whether the path has suitable + * pathkeys. + */ + add_grouped_path(root, rel_grouped, path, AGG_SORTED, agg_info); + + /* + * Repeated creation of hash table (for new parameter values) should + * be possible, does not sound like a good idea in terms of + * efficiency. + */ + if (path->param_info == NULL) + add_grouped_path(root, rel_grouped, path, AGG_HASHED, agg_info); + } + + /* Could not generate any grouped paths? */ + if (rel_grouped->pathlist == NIL) + mark_dummy_rel(rel_grouped); +} + +/* + * Apply partial aggregation to a subpath and add the AggPath to the pathlist. + */ +static void +add_grouped_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, + AggStrategy aggstrategy, RelAggInfo *agg_info) +{ + Path *agg_path; + + + if (aggstrategy == AGG_HASHED) + agg_path = (Path *) create_agg_hashed_path(root, rel, subpath, + agg_info); + else if (aggstrategy == AGG_SORTED) + agg_path = (Path *) create_agg_sorted_path(root, rel, subpath, + agg_info); + else + elog(ERROR, "unexpected strategy %d", aggstrategy); + + /* Add the grouped path to the list of grouped base paths. */ + if (agg_path != NULL) + add_path(rel, (Path *) agg_path); +} + +/* * make_rel_from_joinlist * Build access paths using a "joinlist" to guide the join path search. * @@ -2770,6 +2848,34 @@ make_rel_from_joinlist(PlannerInfo *root, List *joinlist) int varno = ((RangeTblRef *) jlnode)->rtindex; thisrel = find_base_rel(root, varno); + + /* + * Create a grouped relation to facilitate aggregate push-down. + * This makes no sense if thisrel is the only relation of the + * query. + */ + if (bms_nonempty_difference(root->all_baserels, thisrel->relids)) + { + RelOptInfo *rel_grouped; + RelAggInfo *agg_info; + + /* + * Build grouped relation if thisrel is suitable for partial + * aggregation. + */ + rel_grouped = build_simple_grouped_rel(root, varno, &agg_info); + + if (rel_grouped) + { + /* Make the relation available for joining. */ + add_grouped_rel(root, rel_grouped, agg_info); + + /* Add the aggregation paths to it. */ + generate_grouping_paths(root, rel_grouped, thisrel, + agg_info); + set_cheapest(rel_grouped); + } + } } else if (IsA(jlnode, List)) { @@ -2871,6 +2977,7 @@ standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels) for (lev = 2; lev <= levels_needed; lev++) { + RelOptInfo *rel_grouped; ListCell *lc; /* @@ -2907,6 +3014,11 @@ standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels) /* Find and save the cheapest paths for this rel */ set_cheapest(rel); + /* The same for grouped relation if one exists. */ + rel_grouped = find_grouped_rel(root, rel->relids, NULL); + if (rel_grouped) + set_cheapest(rel_grouped); + #ifdef OPTIMIZER_DEBUG debug_print_rel(root, rel); #endif diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 3a9a994733..9d7fa19970 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -4414,7 +4414,6 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) 0, JOIN_INNER, NULL); - rel->rows = clamp_row_est(nrows); cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); @@ -5386,11 +5385,11 @@ set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target) foreach(lc, target->exprs) { Node *node = (Node *) lfirst(lc); + int32 item_width; if (IsA(node, Var)) { Var *var = (Var *) node; - int32 item_width; /* We should not see any upper-level Vars here */ Assert(var->varlevelsup == 0); @@ -5421,6 +5420,20 @@ set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target) Assert(item_width > 0); tuple_width += item_width; } + else if (IsA(node, Aggref)) + { + /* + * If the target is evaluated by AggPath, it'll care of cost + * estimate. If the target is above AggPath (typically target of a + * join relation that contains grouped relation), the cost of + * Aggref should not be accounted for again. + * + * On the other hand, width is always needed. + */ + item_width = get_typavgwidth(exprType(node), exprTypmod(node)); + Assert(item_width > 0); + tuple_width += item_width; + } else { /* diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 78d076b13c..7852cf60c4 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -2535,6 +2535,140 @@ is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist) } /* + * translate_expression_to_rels + * If the appropriate equivalence classes exist, replace vars in + * gvi->gvexpr with vars whose varno is equal to relid. Return NULL if + * translation is not possible or needed. + * + * Note: Currently we only translate Var expressions. This is subject to + * change as the aggregate push-down feature gets enhanced. + */ +GroupedVarInfo * +translate_expression_to_rels(PlannerInfo *root, GroupedVarInfo *gvi, + Index relid) +{ + Var *var; + ListCell *l1; + bool found_orig = false; + Var *var_translated = NULL; + GroupedVarInfo *result; + + /* Can't do anything w/o equivalence classes. */ + if (root->eq_classes == NIL) + return NULL; + + var = castNode(Var, gvi->gvexpr); + + /* + * Do we need to translate the var? + */ + if (var->varno == relid) + return NULL; + + /* + * Find the replacement var. + */ + foreach(l1, root->eq_classes) + { + EquivalenceClass *ec = lfirst_node(EquivalenceClass, l1); + ListCell *l2; + + /* TODO Check if any other EC kind should be ignored. */ + if (ec->ec_has_volatile || ec->ec_below_outer_join || ec->ec_broken) + continue; + + /* Single-element EC can hardly help in translations. */ + if (list_length(ec->ec_members) == 1) + continue; + + /* + * Collect all vars of this EC and their varnos. + * + * ec->ec_relids does not help because we're only interested in a + * subset of EC members. + */ + foreach(l2, ec->ec_members) + { + EquivalenceMember *em = lfirst_node(EquivalenceMember, l2); + Var *ec_var; + + /* + * The grouping expressions derived here are used to evaluate + * possibility to push aggregation down to RELOPT_BASEREL or + * RELOPT_JOINREL relations, and to construct reltargets for the + * grouped rels. We're not interested at the moment whether the + * relations do have children. + */ + if (em->em_is_child) + continue; + + if (!IsA(em->em_expr, Var)) + continue; + + ec_var = castNode(Var, em->em_expr); + if (equal(ec_var, var)) + found_orig = true; + else if (ec_var->varno == relid) + var_translated = ec_var; + + if (found_orig && var_translated) + { + /* + * The replacement Var must have the same data type, otherwise + * the values are not guaranteed to be grouped in the same way + * as values of the original Var. + */ + if (ec_var->vartype != var->vartype) + return NULL; + + break; + } + } + + if (found_orig) + { + /* + * The same expression probably does not exist in multiple ECs. + */ + if (var_translated == NULL) + { + /* + * Failed to translate the expression. + */ + return NULL; + } + else + { + /* Success. */ + break; + } + } + else + { + /* + * Vars of the requested relid can be in the next ECs too. + */ + var_translated = NULL; + } + } + + if (!found_orig) + return NULL; + + result = makeNode(GroupedVarInfo); + memcpy(result, gvi, sizeof(GroupedVarInfo)); + + /* + * translate_expression_to_rels_mutator updates gv_eval_at. + */ + result->gv_eval_at = bms_make_singleton(relid); + result->gvexpr = (Expr *) var_translated; + result->derived = true; + + return result; +} + +/* * is_redundant_with_indexclauses * Test whether rinfo is redundant with any clause in the IndexClause * list. Here, for convenience, we test both simple identity and diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index f680658cf7..d0454646c7 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -22,6 +22,7 @@ #include "partitioning/partbounds.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/selfuncs.h" static void make_rels_by_clause_joins(PlannerInfo *root, @@ -37,7 +38,9 @@ static bool restriction_is_constant_false(List *restrictlist, RelOptInfo *joinrel, bool only_pushed_down); static RelOptInfo *make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, - RelOptInfo *rel2); + RelOptInfo *rel2, + RelAggInfo *agg_info, + RelOptInfo *rel_agg_input); static void populate_joinrel_with_paths(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, RelOptInfo *joinrel, SpecialJoinInfo *sjinfo, List *restrictlist); @@ -670,9 +673,17 @@ join_is_legal(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, /* * make_join_rel_common * The workhorse of make_join_rel(). - */ + * + * 'agg_info' contains the reltarget of grouped relation and everything we + * need to aggregate the join result. If NULL, then the join relation should + * not be grouped. + * + * 'rel_agg_input' describes the AggPath input relation if the join output + * should be aggregated. If NULL is passed, do not aggregate the join output. + */ static RelOptInfo * -make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) +make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, + RelAggInfo *agg_info, RelOptInfo *rel_agg_input) { Relids joinrelids; SpecialJoinInfo *sjinfo; @@ -733,7 +744,7 @@ make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) * goes with this particular joining. */ joinrel = build_join_rel(root, joinrelids, rel1, rel2, sjinfo, - &restrictlist); + &restrictlist, agg_info); /* * If we've already proven this join is empty, we needn't consider any @@ -746,8 +757,26 @@ make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) } /* Add paths to the join relation. */ - populate_joinrel_with_paths(root, rel1, rel2, joinrel, sjinfo, - restrictlist); + if (rel_agg_input == NULL) + { + /* + * Simply join the input relations, whether both are plain or one of + * them is grouped. + */ + populate_joinrel_with_paths(root, rel1, rel2, joinrel, sjinfo, + restrictlist); + } + else + { + /* The join relation is grouped. */ + Assert(agg_info != NULL); + + /* + * Apply partial aggregation to the paths of rel_agg_input and add the + * resulting paths to joinrel. + */ + generate_grouping_paths(root, joinrel, rel_agg_input, agg_info); + } bms_free(joinrelids); @@ -755,6 +784,62 @@ make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) } /* + * make_join_rel_combined + * Join grouped relation to non-grouped one. + */ +static void +make_join_rel_combined(PlannerInfo *root, RelOptInfo *rel1, + RelOptInfo *rel2, + RelAggInfo *agg_info) +{ + RelOptInfo *rel1_grouped; + RelOptInfo *rel2_grouped; + bool rel1_grouped_useful = false; + bool rel2_grouped_useful = false; + + /* Retrieve the grouped relations. */ + rel1_grouped = find_grouped_rel(root, rel1->relids, NULL); + rel2_grouped = find_grouped_rel(root, rel2->relids, NULL); + + /* + * Dummy rel may indicate a join relation that is able to generate grouped + * paths as such (i.e. it has valid agg_info), but for which the path + * actually could not be created (e.g. only AGG_HASHED strategy was + * possible but work_mem was not sufficient for hash table). + */ + rel1_grouped_useful = rel1_grouped != NULL && !IS_DUMMY_REL(rel1_grouped); + rel2_grouped_useful = rel2_grouped != NULL && !IS_DUMMY_REL(rel2_grouped); + + /* Nothing to do if there's no grouped relation. */ + if (!rel1_grouped_useful && !rel2_grouped_useful) + return; + + /* + * At maximum one input rel can be grouped (here we don't care if any rel + * is eventually dummy, the existence of grouped rel indicates that + * aggregates can be pushed down to it). If both were grouped, then + * grouping of one side would change the occurrence of the other side's + * aggregate transient states on the input of the final aggregation. This + * can be handled by adjusting the transient states, but it's not worth + * the effort because it's hard to find a use case for this kind of join. + * + * XXX If the join of two grouped rels is implemented someday, note that + * both rels can have aggregates, so it'd be hard to join grouped rel to + * non-grouped here: 1) such a "mixed join" would require a special + * target, 2) both AGGSPLIT_FINAL_DESERIAL and AGGSPLIT_SIMPLE aggregates + * could appear in the target of the final aggregation node, originating + * from the grouped and the non-grouped input rel respectively. + */ + if (rel1_grouped && rel2_grouped) + return; + + if (rel1_grouped_useful) + make_join_rel_common(root, rel1_grouped, rel2, agg_info, NULL); + else if (rel2_grouped_useful) + make_join_rel_common(root, rel1, rel2_grouped, agg_info, NULL); +} + +/* * make_join_rel * Find or create a join RelOptInfo that represents the join of * the two given rels, and add to it path information for paths @@ -762,14 +847,83 @@ make_join_rel_common(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) * (The join rel may already contain paths generated from other * pairs of rels that add up to the same set of base rels.) * + * In addition to creating an ordinary join relation, try to create a + * grouped one. There are two strategies to achieve that: join a grouped + * relation to plain one, or join two plain relations and apply partial + * aggregation to the result. + * * NB: will return NULL if attempted join is not valid. This can happen when * working with outer joins, or with IN or EXISTS clauses that have been - * turned into joins. + * turned into joins. Besides that, NULL is also returned if caller is + * interested in a grouped relation but it could not be created. + * + * Only the plain relation is returned; if grouped relation exists, it can be + * retrieved using find_grouped_rel(). */ RelOptInfo * make_join_rel(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2) { - return make_join_rel_common(root, rel1, rel2); + Relids joinrelids; + RelAggInfo *agg_info = NULL; + RelOptInfo *joinrel, + *joinrel_plain; + + /* 1) form the plain join. */ + joinrel = make_join_rel_common(root, rel1, rel2, NULL, NULL); + joinrel_plain = joinrel; + + if (joinrel_plain == NULL) + return joinrel_plain; + + /* + * We're done if there are no grouping expressions nor aggregates. + */ + if (root->grouped_var_list == NIL) + return joinrel_plain; + + joinrelids = bms_union(rel1->relids, rel2->relids); + joinrel = find_grouped_rel(root, joinrelids, &agg_info); + + if (joinrel != NULL) + { + /* + * If the same grouped joinrel was already formed, just with the base + * rels divided between rel1 and rel2 in a different way, the matching + * agg_info should already be there. + */ + Assert(agg_info != NULL); + } + else + { + /* + * agg_info must be created from scratch. + */ + agg_info = create_rel_agg_info(root, joinrel_plain); + + /* Cannot we build grouped join? */ + if (agg_info == NULL) + return joinrel_plain; + + /* + * The number of aggregate input rows is simply the number of rows of + * the non-grouped relation, which should have been estimated by now. + */ + agg_info->input_rows = joinrel_plain->rows; + } + + /* + * 2) join two plain rels and aggregate the join paths. Aggregate + * push-down only makes sense if the join is not the top-level one. + */ + if (bms_nonempty_difference(root->all_baserels, joinrelids)) + make_join_rel_common(root, rel1, rel2, agg_info, joinrel_plain); + + /* + * 3) combine plain and grouped relations. + */ + make_join_rel_combined(root, rel1, rel2, agg_info); + + return joinrel_plain; } /* diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index 73da0c2d8e..9f97a2dd8f 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -48,6 +48,8 @@ typedef struct PostponedQual } PostponedQual; +static void create_aggregate_grouped_var_infos(PlannerInfo *root); +static void create_grouping_expr_grouped_var_infos(PlannerInfo *root); static void extract_lateral_references(PlannerInfo *root, RelOptInfo *brel, Index rtindex); static List *deconstruct_recurse(PlannerInfo *root, Node *jtnode, @@ -273,6 +275,261 @@ add_vars_to_targetlist(PlannerInfo *root, List *vars, } } +/* + * Add GroupedVarInfo to grouped_var_list for each aggregate as well as for + * each possible grouping expression. + * + * root->group_pathkeys must be setup before this function is called. + */ +extern void +setup_aggregate_pushdown(PlannerInfo *root) +{ + ListCell *lc; + + /* + * Isn't user interested in the aggregate push-down feature? + */ + if (!enable_agg_pushdown) + return; + + /* The feature can only be applied to grouped aggregation. */ + if (!root->parse->groupClause) + return; + + /* + * Grouping sets require multiple different groupings but the base + * relation can only generate one. + */ + if (root->parse->groupingSets) + return; + + /* + * SRF is not allowed in the aggregate argument and we don't even want it + * in the GROUP BY clause, so forbid it in general. It needs to be + * analyzed if evaluation of a GROUP BY clause containing SRF below the + * query targetlist would be correct. Currently it does not seem to be an + * important use case. + */ + if (root->parse->hasTargetSRFs) + return; + + /* Create GroupedVarInfo per (distinct) aggregate. */ + create_aggregate_grouped_var_infos(root); + + /* Isn't there any aggregate to be pushed down? */ + if (root->grouped_var_list == NIL) + return; + + /* Create GroupedVarInfo per grouping expression. */ + create_grouping_expr_grouped_var_infos(root); + + /* Isn't there any useful grouping expression for aggregate push-down? */ + if (root->grouped_var_list == NIL) + return; + + /* + * Now that we know that grouping can be pushed down, search for the + * maximum sortgroupref. The base relations may need it if extra grouping + * expressions get added to them. + */ + Assert(root->max_sortgroupref == 0); + foreach(lc, root->processed_tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, lc); + + if (te->ressortgroupref > root->max_sortgroupref) + root->max_sortgroupref = te->ressortgroupref; + } +} + +/* + * Create GroupedVarInfo for each distinct aggregate. + * + * If any aggregate is not suitable, set root->grouped_var_list to NIL and + * return. + */ +static void +create_aggregate_grouped_var_infos(PlannerInfo *root) +{ + List *tlist_exprs; + ListCell *lc; + + Assert(root->grouped_var_list == NIL); + + tlist_exprs = pull_var_clause((Node *) root->processed_tlist, + PVC_INCLUDE_AGGREGATES | + PVC_INCLUDE_WINDOWFUNCS); + + /* + * Although GroupingFunc is related to root->parse->groupingSets, this + * field does not necessarily reflect its presence. + */ + foreach(lc, tlist_exprs) + { + Expr *expr = (Expr *) lfirst(lc); + + if (IsA(expr, GroupingFunc)) + return; + } + + /* + * Aggregates within the HAVING clause need to be processed in the same + * way as those in the main targetlist. + * + * Note that the contained aggregates will be pushed down, but the + * containing HAVING clause must be ignored until the aggregation is + * finalized. + */ + if (root->parse->havingQual != NULL) + { + List *having_exprs; + + having_exprs = pull_var_clause((Node *) root->parse->havingQual, + PVC_INCLUDE_AGGREGATES); + if (having_exprs != NIL) + tlist_exprs = list_concat(tlist_exprs, having_exprs); + } + + if (tlist_exprs == NIL) + return; + + foreach(lc, tlist_exprs) + { + Expr *expr = (Expr *) lfirst(lc); + Aggref *aggref; + ListCell *lc2; + GroupedVarInfo *gvi; + bool exists; + + /* + * tlist_exprs may also contain Vars or WindowFuncs, but we only need + * Aggrefs. + */ + if (IsA(expr, Var) ||IsA(expr, WindowFunc)) + continue; + + aggref = castNode(Aggref, expr); + + /* TODO Think if (some of) these can be handled. */ + if (aggref->aggvariadic || + aggref->aggdirectargs || aggref->aggorder || + aggref->aggdistinct) + { + /* + * Aggregation push-down is not useful if at least one aggregate + * cannot be evaluated below the top-level join. + * + * XXX Is it worth freeing the GroupedVarInfos and their subtrees? + */ + root->grouped_var_list = NIL; + break; + } + + /* Does GroupedVarInfo for this aggregate already exist? */ + exists = false; + foreach(lc2, root->grouped_var_list) + { + gvi = lfirst_node(GroupedVarInfo, lc2); + + if (equal(expr, gvi->gvexpr)) + { + exists = true; + break; + } + } + + /* Construct a new GroupedVarInfo if does not exist yet. */ + if (!exists) + { + Relids relids; + + gvi = makeNode(GroupedVarInfo); + gvi->gvexpr = (Expr *) copyObject(aggref); + + /* Find out where the aggregate should be evaluated. */ + relids = pull_varnos((Node *) aggref); + if (!bms_is_empty(relids)) + gvi->gv_eval_at = relids; + else + gvi->gv_eval_at = NULL; + + root->grouped_var_list = lappend(root->grouped_var_list, gvi); + } + } + + list_free(tlist_exprs); +} + +/* + * Create GroupedVarInfo for each expression usable as grouping key. + * + * In addition to the expressions of the query targetlist, group_pathkeys is + * also considered the source of grouping expressions. That increases the + * chance to get the relation output grouped. + */ +static void +create_grouping_expr_grouped_var_infos(PlannerInfo *root) +{ + ListCell *l1, + *l2; + List *exprs = NIL; + List *sortgrouprefs = NIL; + + /* + * Make sure GroupedVarInfo exists for each expression usable as grouping + * key. + */ + foreach(l1, root->parse->groupClause) + { + SortGroupClause *sgClause; + TargetEntry *te; + Index sortgroupref; + + sgClause = lfirst_node(SortGroupClause, l1); + te = get_sortgroupclause_tle(sgClause, root->processed_tlist); + sortgroupref = te->ressortgroupref; + + Assert(sortgroupref > 0); + + /* + * Non-zero sortgroupref does not necessarily imply grouping + * expression: data can also be sorted by aggregate. + */ + if (IsA(te->expr, Aggref)) + continue; + + /* + * The aggregate push-down feature currently supports only plain Vars + * as grouping expressions. + */ + if (!IsA(te->expr, Var)) + { + root->grouped_var_list = NIL; + return; + } + + exprs = lappend(exprs, te->expr); + sortgrouprefs = lappend_int(sortgrouprefs, sortgroupref); + } + + /* + * Construct GroupedVarInfo for each expression. + */ + forboth(l1, exprs, l2, sortgrouprefs) + { + Var *var = lfirst_node(Var, l1); + int sortgroupref = lfirst_int(l2); + GroupedVarInfo *gvi = makeNode(GroupedVarInfo); + + gvi->gvexpr = (Expr *) copyObject(var); + gvi->sortgroupref = sortgroupref; + + /* Find out where the expression should be evaluated. */ + gvi->gv_eval_at = bms_make_singleton(var->varno); + + root->grouped_var_list = lappend(root->grouped_var_list, gvi); + } +} /***************************************************************************** * diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 9381939c82..c4d07dfac4 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -442,7 +442,7 @@ build_minmax_path(PlannerInfo *root, MinMaxAggInfo *mminfo, subroot->tuple_fraction = 1.0; subroot->limit_tuples = 1.0; - final_rel = query_planner(subroot, minmax_qp_callback, NULL); + final_rel = query_planner(subroot, minmax_qp_callback, NULL, NULL); /* * Since we didn't go through subquery_planner() to handle the subquery, diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 0b9999c8a6..4103a393e6 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -45,6 +45,9 @@ * qp_callback is a function to compute query_pathkeys once it's safe to do so * qp_extra is optional extra data to pass to qp_callback * + * If final_rel_grouped_p is valid, relation containing grouped paths may be + * saved to *final_rel_grouped_p. + * * Note: the PlannerInfo node also includes a query_pathkeys field, which * tells query_planner the sort order that is desired in the final output * plan. This value is *not* available at call time, but is computed by @@ -53,7 +56,8 @@ */ RelOptInfo * query_planner(PlannerInfo *root, - query_pathkeys_callback qp_callback, void *qp_extra) + query_pathkeys_callback qp_callback, void *qp_extra, + RelOptInfo **final_rel_grouped_p) { Query *parse = root->parse; List *joinlist; @@ -66,6 +70,8 @@ query_planner(PlannerInfo *root, * here. */ root->join_rel_list = makeNode(RelInfoList); + root->grouped_rel_list = makeNode(RelInfoList); + root->agg_info_list = makeNode(RelInfoList); root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; @@ -74,6 +80,7 @@ query_planner(PlannerInfo *root, root->full_join_clauses = NIL; root->join_info_list = NIL; root->placeholder_list = NIL; + root->grouped_var_list = NIL; root->fkey_list = NIL; root->initial_rels = NIL; @@ -255,6 +262,16 @@ query_planner(PlannerInfo *root, extract_restriction_or_clauses(root); /* + * If the query result can be grouped, check if any grouping can be + * performed below the top-level join. If so, setup + * root->grouped_var_list. + * + * The base relations should be fully initialized now, so that we have + * enough info to decide whether grouping is possible. + */ + setup_aggregate_pushdown(root); + + /* * Now expand appendrels by adding "otherrels" for their children. We * delay this to the end so that we have as much information as possible * available for each baserel, including all restriction clauses. That @@ -274,5 +291,9 @@ query_planner(PlannerInfo *root, final_rel->cheapest_total_path->param_info != NULL) elog(ERROR, "failed to construct the join relation"); + if (final_rel_grouped_p) + *final_rel_grouped_p = find_grouped_rel(root, final_rel->relids, + NULL); + return final_rel; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index ca3b7f29e1..aa18fe5694 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -151,6 +151,7 @@ static double get_number_of_groups(PlannerInfo *root, List *target_list); static RelOptInfo *create_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, + RelOptInfo *input_rel_grouped, PathTarget *target, bool target_parallel_safe, const AggClauseCosts *agg_costs, @@ -164,6 +165,7 @@ static RelOptInfo *make_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, Node *havingQual); static void create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, + RelOptInfo *input_rel_grouped, RelOptInfo *grouped_rel, const AggClauseCosts *agg_costs, grouping_sets_data *gd, @@ -623,6 +625,7 @@ subquery_planner(PlannerGlobal *glob, Query *parse, memset(root->upper_rels, 0, sizeof(root->upper_rels)); memset(root->upper_targets, 0, sizeof(root->upper_targets)); root->processed_tlist = NIL; + root->max_sortgroupref = 0; root->grouping_map = NULL; root->minmax_aggs = NIL; root->qual_security_level = 0; @@ -1943,6 +1946,7 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, List *activeWindows = NIL; grouping_sets_data *gset_data = NULL; standard_qp_extra qp_extra; + RelOptInfo *current_rel_grouped = NULL; /* A recursive query should always have setOperations */ Assert(!root->hasRecursion); @@ -2045,7 +2049,9 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, * We also generate (in standard_qp_callback) pathkey representations * of the query's sort clause, distinct clause, etc. */ - current_rel = query_planner(root, standard_qp_callback, &qp_extra); + current_rel = query_planner(root, + standard_qp_callback, &qp_extra, + ¤t_rel_grouped); /* * Convert the query's result tlist into PathTarget format. @@ -2190,6 +2196,7 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, { current_rel = create_grouping_paths(root, current_rel, + current_rel_grouped, grouping_target, grouping_target_parallel_safe, &agg_costs, @@ -3798,6 +3805,7 @@ get_number_of_groups(PlannerInfo *root, static RelOptInfo * create_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, + RelOptInfo *input_rel_grouped, PathTarget *target, bool target_parallel_safe, const AggClauseCosts *agg_costs, @@ -3888,7 +3896,8 @@ create_grouping_paths(PlannerInfo *root, else extra.patype = PARTITIONWISE_AGGREGATE_NONE; - create_ordinary_grouping_paths(root, input_rel, grouped_rel, + create_ordinary_grouping_paths(root, input_rel, input_rel_grouped, + grouped_rel, agg_costs, gd, &extra, &partially_grouped_rel); } @@ -4046,6 +4055,7 @@ create_degenerate_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, */ static void create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, + RelOptInfo *input_rel_grouped, RelOptInfo *grouped_rel, const AggClauseCosts *agg_costs, grouping_sets_data *gd, @@ -4093,13 +4103,23 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, if ((extra->flags & GROUPING_CAN_PARTIAL_AGG) != 0) { bool force_rel_creation; + bool have_agg_pushdown_paths; /* - * If we're doing partitionwise aggregation at this level, force - * creation of a partially_grouped_rel so we can add partitionwise - * paths to it. + * Check if the aggregate push-down feature succeeded to generate any + * paths. Dummy relation can appear here because grouped paths are not + * guaranteed to exist for a relation. */ - force_rel_creation = (patype == PARTITIONWISE_AGGREGATE_PARTIAL); + have_agg_pushdown_paths = input_rel_grouped != NULL && + !IS_DUMMY_REL(input_rel_grouped); + + /* + * If we're doing partitionwise aggregation at this level or if + * aggregate push-down succeeded to create some paths, force creation + * of a partially_grouped_rel so we can add the related paths to it. + */ + force_rel_creation = patype == PARTITIONWISE_AGGREGATE_PARTIAL || + have_agg_pushdown_paths; partially_grouped_rel = create_partial_grouping_paths(root, @@ -4108,6 +4128,23 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, gd, extra, force_rel_creation); + + /* + * No further processing is needed for paths provided by the aggregate + * push-down feature. Simply add them to the partially grouped + * relation. + */ + if (have_agg_pushdown_paths) + { + ListCell *lc; + + foreach(lc, input_rel_grouped->pathlist) + { + Path *path = (Path *) lfirst(lc); + + add_path(partially_grouped_rel, path); + } + } } /* Set out parameter. */ @@ -4132,10 +4169,14 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, /* Gather any partially grouped partial paths. */ if (partially_grouped_rel && partially_grouped_rel->partial_pathlist) - { gather_grouping_paths(root, partially_grouped_rel); + + /* + * The non-partial paths can come either from the Gather above or from + * aggregate push-down. + */ + if (partially_grouped_rel && partially_grouped_rel->pathlist) set_cheapest(partially_grouped_rel); - } /* * Estimate number of groups. @@ -7296,6 +7337,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root, /* Create grouping paths for this child relation. */ create_ordinary_grouping_paths(root, child_input_rel, + NULL, child_grouped_rel, agg_costs, gd, &child_extra, &child_partially_grouped_rel); diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index dc11f098e0..3810345b3d 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -2380,6 +2380,39 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) /* No referent found for Var */ elog(ERROR, "variable not found in subplan target lists"); } + if (IsA(node, Aggref)) + { + Aggref *aggref = castNode(Aggref, node); + + /* + * The upper plan targetlist can contain Aggref whose value has + * already been evaluated by the subplan. However this can only happen + * with specific value of aggsplit. + */ + if (aggref->aggsplit == AGGSPLIT_INITIAL_SERIAL) + { + /* See if the Aggref has bubbled up from a lower plan node */ + if (context->outer_itlist && context->outer_itlist->has_non_vars) + { + newvar = search_indexed_tlist_for_non_var((Expr *) node, + context->outer_itlist, + OUTER_VAR); + if (newvar) + return (Node *) newvar; + } + if (context->inner_itlist && context->inner_itlist->has_non_vars) + { + newvar = search_indexed_tlist_for_non_var((Expr *) node, + context->inner_itlist, + INNER_VAR); + if (newvar) + return (Node *) newvar; + } + } + + /* No referent found for Aggref */ + elog(ERROR, "Aggref not found in subplan target lists"); + } if (IsA(node, PlaceHolderVar)) { PlaceHolderVar *phv = (PlaceHolderVar *) node; diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index e20bee0b33..a6ca35ad98 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -891,6 +891,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, memset(subroot->upper_rels, 0, sizeof(subroot->upper_rels)); memset(subroot->upper_targets, 0, sizeof(subroot->upper_targets)); subroot->processed_tlist = NIL; + root->max_sortgroupref = 0; subroot->grouping_map = NULL; subroot->minmax_aggs = NIL; subroot->qual_security_level = 0; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 0ac73984d2..4126caac8c 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -2528,8 +2528,7 @@ create_projection_path(PlannerInfo *root, pathnode->path.pathtype = T_Result; pathnode->path.parent = rel; pathnode->path.pathtarget = target; - /* For now, assume we are above any joins, so no parameterization */ - pathnode->path.param_info = NULL; + pathnode->path.param_info = subpath->param_info; pathnode->path.parallel_aware = false; pathnode->path.parallel_safe = rel->consider_parallel && subpath->parallel_safe && @@ -2969,6 +2968,146 @@ create_agg_path(PlannerInfo *root, } /* + * Apply AGG_SORTED aggregation path to subpath if it's suitably sorted. + * + * NULL is returned if sorting of subpath output is not suitable. + */ +AggPath * +create_agg_sorted_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, + RelAggInfo *agg_info) +{ + Node *agg_exprs; + AggSplit aggsplit; + AggClauseCosts agg_costs; + PathTarget *target; + double dNumGroups; + ListCell *lc1; + List *key_subset = NIL; + AggPath *result = NULL; + + aggsplit = AGGSPLIT_INITIAL_SERIAL; + agg_exprs = (Node *) agg_info->agg_exprs; + target = agg_info->target; + + if (subpath->pathkeys == NIL) + return NULL; + + if (!grouping_is_sortable(root->parse->groupClause)) + return NULL; + + /* + * Find all query pathkeys that our relation does affect. + */ + foreach(lc1, root->group_pathkeys) + { + PathKey *gkey = castNode(PathKey, lfirst(lc1)); + ListCell *lc2; + + foreach(lc2, subpath->pathkeys) + { + PathKey *skey = castNode(PathKey, lfirst(lc2)); + + if (skey == gkey) + { + key_subset = lappend(key_subset, gkey); + break; + } + } + } + + if (key_subset == NIL) + return NULL; + + /* Check if AGG_SORTED is useful for the whole query. */ + if (!pathkeys_contained_in(key_subset, subpath->pathkeys)) + return NULL; + + MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); + get_agg_clause_costs(root, (Node *) agg_exprs, aggsplit, &agg_costs); + + Assert(agg_info->group_exprs != NIL); + dNumGroups = estimate_num_groups(root, agg_info->group_exprs, + subpath->rows, NULL); + + /* + * qual is NIL because the HAVING clause cannot be evaluated until the + * final value of the aggregate is known. + */ + result = create_agg_path(root, rel, subpath, target, + AGG_SORTED, aggsplit, + agg_info->group_clauses, + NIL, + &agg_costs, + dNumGroups); + + /* The agg path should require no fewer parameters than the plain one. */ + result->path.param_info = subpath->param_info; + + return result; +} + +/* + * Apply AGG_HASHED aggregation to subpath. + */ +AggPath * +create_agg_hashed_path(PlannerInfo *root, RelOptInfo *rel, + Path *subpath, RelAggInfo *agg_info) +{ + bool can_hash; + Node *agg_exprs; + AggSplit aggsplit; + AggClauseCosts agg_costs; + PathTarget *target; + double dNumGroups; + double hashaggtablesize; + Query *parse = root->parse; + AggPath *result = NULL; + + /* Do not try to create hash table for each parameter value. */ + Assert(subpath->param_info == NULL); + + aggsplit = AGGSPLIT_INITIAL_SERIAL; + agg_exprs = (Node *) agg_info->agg_exprs; + target = agg_info->target; + + MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); + get_agg_clause_costs(root, agg_exprs, aggsplit, &agg_costs); + + can_hash = (parse->groupClause != NIL && + parse->groupingSets == NIL && + agg_costs.numOrderedAggs == 0 && + grouping_is_hashable(parse->groupClause)); + + if (can_hash) + { + Assert(agg_info->group_exprs != NIL); + dNumGroups = estimate_num_groups(root, agg_info->group_exprs, + subpath->rows, NULL); + + hashaggtablesize = estimate_hashagg_tablesize(subpath, &agg_costs, + dNumGroups); + + if (hashaggtablesize < work_mem * 1024L) + { + /* + * qual is NIL because the HAVING clause cannot be evaluated until + * the final value of the aggregate is known. + */ + result = create_agg_path(root, rel, subpath, + target, + AGG_HASHED, + aggsplit, + agg_info->group_clauses, + NIL, + &agg_costs, + dNumGroups); + } + } + + return result; +} + +/* * create_groupingsets_path * Creates a pathnode that represents performing GROUPING SETS aggregation * diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index c238dd6538..30ccbde60f 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -17,18 +17,24 @@ #include <limits.h> #include "miscadmin.h" +#include "catalog/pg_class_d.h" +#include "catalog/pg_constraint.h" #include "optimizer/appendinfo.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" +#include "optimizer/optimizer.h" #include "optimizer/inherit.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/placeholder.h" #include "optimizer/plancat.h" +#include "optimizer/planner.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" +#include "parser/parse_oper.h" #include "partitioning/partbounds.h" #include "utils/hsearch.h" +#include "utils/selfuncs.h" typedef struct RelInfoEntry @@ -63,6 +69,9 @@ static void build_child_join_reltarget(PlannerInfo *root, RelOptInfo *childrel, int nappinfos, AppendRelInfo **appinfos); +static bool init_grouping_targets(PlannerInfo *root, RelOptInfo *rel, + PathTarget *target, PathTarget *agg_input, + List *gvis, List **group_exprs_extra_p); /* @@ -352,6 +361,102 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) } /* + * build_simple_grouped_rel + * Construct a new RelOptInfo for a grouped base relation out of an + * existing non-grouped relation. On success, pointer to the corresponding + * RelAggInfo is stored in *agg_info_p in addition to returning the grouped + * relation. + */ +RelOptInfo * +build_simple_grouped_rel(PlannerInfo *root, int relid, + RelAggInfo **agg_info_p) +{ + RangeTblEntry *rte; + RelOptInfo *rel_plain, + *rel_grouped; + RelAggInfo *agg_info; + + /* Isn't there any grouping expression to be pushed down? */ + if (root->grouped_var_list == NIL) + return NULL; + + rel_plain = root->simple_rel_array[relid]; + + /* Caller should only pass rti that represents base relation. */ + Assert(rel_plain != NULL); + + /* + * Not all RTE kinds are supported when grouping is considered. + * + * TODO Consider relaxing some of these restrictions. + */ + rte = root->simple_rte_array[rel_plain->relid]; + if (rte->rtekind != RTE_RELATION || + rte->relkind == RELKIND_FOREIGN_TABLE || + rte->tablesample != NULL) + return NULL; + + /* + * Grouped append relation is not supported yet. + */ + if (rte->inh) + return NULL; + + /* + * Currently we do not support child relations ("other rels"). + */ + if (rel_plain->reloptkind != RELOPT_BASEREL) + return NULL; + + /* + * Prepare the information we need for aggregation of the rel contents. + */ + agg_info = create_rel_agg_info(root, rel_plain); + if (agg_info == NULL) + return NULL; + + /* + * TODO Consider if 1) a flat copy is o.k., 2) it's safer in terms of + * adding new fields to RelOptInfo) to copy everything and then reset some + * fields, or to zero the structure and copy individual fields. + */ + rel_grouped = makeNode(RelOptInfo); + memcpy(rel_grouped, rel_plain, sizeof(RelOptInfo)); + + /* + * Note on consider_startup: while the AGG_HASHED strategy needs the whole + * relation, AGG_SORTED does not. Therefore we do not force + * consider_startup to false. + */ + + /* + * Set the appropriate target for grouped paths. + * + * reltarget should match the target of partially aggregated paths. + */ + rel_grouped->reltarget = agg_info->target; + + /* + * Grouped paths must not be mixed with the plain ones. + */ + rel_grouped->pathlist = NIL; + rel_grouped->partial_pathlist = NIL; + rel_grouped->cheapest_startup_path = NULL; + rel_grouped->cheapest_total_path = NULL; + rel_grouped->cheapest_unique_path = NULL; + rel_grouped->cheapest_parameterized_paths = NIL; + + /* + * The number of aggregation input rows is simply the number of rows of + * the non-grouped relation, which should have been estimated by now. + */ + agg_info->input_rows = rel_plain->rows; + + *agg_info_p = agg_info; + return rel_grouped; +} + +/* * find_base_rel * Find a base or other relation entry, which must already exist. */ @@ -460,10 +565,14 @@ find_rel_info(RelInfoList *list, Relids relids) foreach(l, list->items) { void *item = lfirst(l); - Relids item_relids; + Relids item_relids = NULL; - Assert(IsA(item, RelOptInfo)); - item_relids = ((RelOptInfo *) item)->relids; + Assert(IsA(item, RelOptInfo) ||IsA(item, RelAggInfo)); + + if (IsA(item, RelOptInfo)) + item_relids = ((RelOptInfo *) item)->relids; + else if (IsA(item, RelAggInfo)) + item_relids = ((RelAggInfo *) item)->relids; if (bms_equal(item_relids, relids)) return item; @@ -492,7 +601,7 @@ find_join_rel(PlannerInfo *root, Relids relids) static void add_rel_info(RelInfoList *list, void *data) { - Assert(IsA(data, RelOptInfo)); + Assert(IsA(data, RelOptInfo) ||IsA(data, RelAggInfo)); /* GEQO requires us to append the new joinrel to the end of the list! */ list->items = lappend(list->items, data); @@ -504,7 +613,11 @@ add_rel_info(RelInfoList *list, void *data) RelInfoEntry *hentry; bool found; - relids = ((RelOptInfo *) data)->relids; + if (IsA(data, RelOptInfo)) + relids = ((RelOptInfo *) data)->relids; + else if (IsA(data, RelAggInfo)) + relids = ((RelAggInfo *) data)->relids; + hentry = (RelInfoEntry *) hash_search(list->hash, &relids, HASH_ENTER, @@ -526,6 +639,57 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) } /* + * add_grouped_rel + * Add grouped base or join relation to the list of grouped relations in + * the given PlannerInfo. Also add the corresponding RelAggInfo to + * agg_info_list. + */ +void +add_grouped_rel(PlannerInfo *root, RelOptInfo *rel, RelAggInfo *agg_info) +{ + add_rel_info(root->grouped_rel_list, rel); + add_rel_info(root->agg_info_list, agg_info); +} + +/* + * find_grouped_rel + * Returns grouped relation entry (base or join relation) corresponding to + * 'relids' or NULL if none exists. + * + * If agg_info_p is a valid pointer, then pointer to RelAggInfo that + * corresponds to the relation returned is assigned to *agg_info_p. + */ +RelOptInfo * +find_grouped_rel(PlannerInfo *root, Relids relids, RelAggInfo **agg_info_p) +{ + RelOptInfo *rel; + + rel = (RelOptInfo *) find_rel_info(root->grouped_rel_list, relids); + if (rel == NULL) + { + if (agg_info_p) + *agg_info_p = NULL; + + return NULL; + } + + /* Is caller interested in RelAggInfo? */ + if (agg_info_p) + { + RelAggInfo *agg_info; + + agg_info = (RelAggInfo *) find_rel_info(root->agg_info_list, relids); + + /* The relation exists, so the agg_info should be there too. */ + Assert(agg_info != NULL); + + *agg_info_p = agg_info; + } + + return rel; +} + +/* * set_foreign_rel_properties * Set up foreign-join fields if outer and inner relation are foreign * tables (or joins) belonging to the same server and assigned to the same @@ -587,6 +751,7 @@ set_foreign_rel_properties(RelOptInfo *joinrel, RelOptInfo *outer_rel, * 'restrictlist_ptr': result variable. If not NULL, *restrictlist_ptr * receives the list of RestrictInfo nodes that apply to this * particular pair of joinable relations. + * 'agg_info' indicates that grouped join relation should be created. * * restrictlist_ptr makes the routine's API a little grotty, but it saves * duplicated calculation of the restrictlist... @@ -597,10 +762,12 @@ build_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, RelOptInfo *inner_rel, SpecialJoinInfo *sjinfo, - List **restrictlist_ptr) + List **restrictlist_ptr, + RelAggInfo *agg_info) { RelOptInfo *joinrel; List *restrictlist; + bool grouped = agg_info != NULL; /* This function should be used only for join between parents. */ Assert(!IS_OTHER_REL(outer_rel) && !IS_OTHER_REL(inner_rel)); @@ -608,7 +775,8 @@ build_join_rel(PlannerInfo *root, /* * See if we already have a joinrel for this set of base rels. */ - joinrel = find_join_rel(root, joinrelids); + joinrel = !grouped ? find_join_rel(root, joinrelids) : + find_grouped_rel(root, joinrelids, NULL); if (joinrel) { @@ -700,9 +868,21 @@ build_join_rel(PlannerInfo *root, * and inner rels we first try to build it from. But the contents should * be the same regardless. */ - build_joinrel_tlist(root, joinrel, outer_rel); - build_joinrel_tlist(root, joinrel, inner_rel); - add_placeholders_to_joinrel(root, joinrel, outer_rel, inner_rel); + if (!grouped) + { + joinrel->reltarget = create_empty_pathtarget(); + build_joinrel_tlist(root, joinrel, outer_rel); + build_joinrel_tlist(root, joinrel, inner_rel); + add_placeholders_to_joinrel(root, joinrel, outer_rel, inner_rel); + } + else + { + /* + * The target for grouped join should already have its cost and width + * computed, see create_rel_agg_info(). + */ + joinrel->reltarget = agg_info->target; + } /* * add_placeholders_to_joinrel also took care of adding the ph_lateral @@ -734,49 +914,73 @@ build_join_rel(PlannerInfo *root, joinrel->has_eclass_joins = has_relevant_eclass_joinclause(root, joinrel); /* Store the partition information. */ - build_joinrel_partition_info(joinrel, outer_rel, inner_rel, restrictlist, - sjinfo->jointype); - - /* - * Set estimates of the joinrel's size. - */ - set_joinrel_size_estimates(root, joinrel, outer_rel, inner_rel, - sjinfo, restrictlist); - - /* - * Set the consider_parallel flag if this joinrel could potentially be - * scanned within a parallel worker. If this flag is false for either - * inner_rel or outer_rel, then it must be false for the joinrel also. - * Even if both are true, there might be parallel-restricted expressions - * in the targetlist or quals. - * - * Note that if there are more than two rels in this relation, they could - * be divided between inner_rel and outer_rel in any arbitrary way. We - * assume this doesn't matter, because we should hit all the same baserels - * and joinclauses while building up to this joinrel no matter which we - * take; therefore, we should make the same decision here however we get - * here. - */ - if (inner_rel->consider_parallel && outer_rel->consider_parallel && - is_parallel_safe(root, (Node *) restrictlist) && - is_parallel_safe(root, (Node *) joinrel->reltarget->exprs)) - joinrel->consider_parallel = true; + if (!grouped) + build_joinrel_partition_info(joinrel, outer_rel, inner_rel, + restrictlist, sjinfo->jointype); /* Add the joinrel to the PlannerInfo. */ - add_join_rel(root, joinrel); + if (!grouped) + add_join_rel(root, joinrel); + else + add_grouped_rel(root, joinrel, agg_info); /* - * Also, if dynamic-programming join search is active, add the new joinrel - * to the appropriate sublist. Note: you might think the Assert on number - * of members should be for equality, but some of the level 1 rels might - * have been joinrels already, so we can only assert <=. + * Also, if dynamic-programming join search is active, add the new + * joinrelset to the appropriate sublist. Note: you might think the + * Assert on number of members should be for equality, but some of the + * level 1 rels might have been joinrels already, so we can only assert + * <=. + * + * Do noting for grouped relation as it's stored aside from + * join_rel_level. */ - if (root->join_rel_level) + if (root->join_rel_level && !grouped) { Assert(root->join_cur_level > 0); - Assert(root->join_cur_level <= bms_num_members(joinrel->relids)); + Assert(root->join_cur_level <= bms_num_members(joinrelids)); root->join_rel_level[root->join_cur_level] = - lappend(root->join_rel_level[root->join_cur_level], joinrel); + lappend(root->join_rel_level[root->join_cur_level], + joinrel); + } + + /* Set estimates of the joinrel's size. */ + if (!grouped) + { + set_joinrel_size_estimates(root, joinrel, outer_rel, inner_rel, + sjinfo, restrictlist); + + /* + * Set the consider_parallel flag if this joinrel could potentially be + * scanned within a parallel worker. If this flag is false for either + * inner_rel or outer_rel, then it must be false for the joinrel also. + * Even if both are true, there might be parallel-restricted + * expressions in the targetlist or quals. + * + * Note that if there are more than two rels in this relation, they + * could be divided between inner_rel and outer_rel in any arbitrary + * way. We assume this doesn't matter, because we should hit all the + * same baserels and joinclauses while building up to this joinrel no + * matter which we take; therefore, we should make the same decision + * here however we get here. + */ + if (inner_rel->consider_parallel && outer_rel->consider_parallel && + is_parallel_safe(root, (Node *) restrictlist) && + is_parallel_safe(root, (Node *) joinrel->reltarget->exprs)) + joinrel->consider_parallel = true; + } + else + { + /* + * Grouping essentially changes the number of rows. + * + * XXX We do not distinguish whether two plain rels are joined and the + * result is aggregated, or the aggregation has been already applied + * to one of the input rels. Is this worth extra effort, e.g. + * maintaining a separate RelOptInfo for each case (one difficulty + * that would introduce is construction of AppendPath)? + */ + joinrel->rows = estimate_num_groups(root, agg_info->group_exprs, + agg_info->input_rows, NULL); } return joinrel; @@ -1820,3 +2024,625 @@ build_child_join_reltarget(PlannerInfo *root, childrel->reltarget->cost.per_tuple = parentrel->reltarget->cost.per_tuple; childrel->reltarget->width = parentrel->reltarget->width; } + +/* + * Check if the relation can produce grouped paths and return the information + * it'll need for it. The passed relation is the non-grouped one which has the + * reltarget already constructed. + */ +RelAggInfo * +create_rel_agg_info(PlannerInfo *root, RelOptInfo *rel) +{ + List *gvis; + List *aggregates = NIL; + bool found_other_rel_agg; + ListCell *lc; + RelAggInfo *result; + PathTarget *agg_input; + PathTarget *target = NULL; + List *grp_exprs_extra = NIL; + List *group_clauses_final; + int i; + + /* + * The function shouldn't have been called if there's no opportunity for + * aggregation push-down. + */ + Assert(root->grouped_var_list != NIL); + + result = makeNode(RelAggInfo); + + /* + * The current implementation of aggregation push-down cannot handle + * PlaceHolderVar (PHV). + * + * If we knew that the PHV should be evaluated in this target (and of + * course, if its expression matched some Aggref argument), we'd just let + * init_grouping_targets add that Aggref. On the other hand, if we knew + * that the PHV is evaluated below the current rel, we could ignore it + * because the referencing Aggref would take care of propagation of the + * value to upper joins. + * + * The problem is that the same PHV can be evaluated in the target of the + * current rel or in that of lower rel --- depending on the input paths. + * For example, consider rel->relids = {A, B, C} and if ph_eval_at = {B, + * C}. Path "A JOIN (B JOIN C)" implies that the PHV is evaluated by the + * "(B JOIN C)", while path "(A JOIN B) JOIN C" evaluates the PHV itself. + */ + foreach(lc, rel->reltarget->exprs) + { + Expr *expr = lfirst(lc); + + if (IsA(expr, PlaceHolderVar)) + return NULL; + } + + if (IS_SIMPLE_REL(rel)) + { + RangeTblEntry *rte = root->simple_rte_array[rel->relid];; + + /* + * rtekind != RTE_RELATION case is not supported yet. + */ + if (rte->rtekind != RTE_RELATION) + return NULL; + } + + /* Caller should only pass base relations or joins. */ + Assert(rel->reloptkind == RELOPT_BASEREL || + rel->reloptkind == RELOPT_JOINREL); + + /* + * If any outer join can set the attribute value to NULL, the Agg plan + * would receive different input at the base rel level. + * + * XXX For RELOPT_JOINREL, do not return if all the joins that can set any + * entry of the grouped target (do we need to postpone this check until + * the grouped target is available, and init_grouping_targets take care?) + * of this rel to NULL are provably below rel. (It's ok if rel is one of + * these joins.) + */ + if (bms_overlap(rel->relids, root->nullable_baserels)) + return NULL; + + /* + * Use equivalence classes to generate additional grouping expressions for + * the current rel. Without these we might not be able to apply + * aggregation to the relation result set. + * + * It's important that create_grouping_expr_grouped_var_infos has + * processed the explicit grouping columns by now. If the grouping clause + * contains multiple expressions belonging to the same EC, the original + * (i.e. not derived) one should be preferred when we build grouping + * target for a relation. Otherwise we have a problem when trying to match + * target entries to grouping clauses during plan creation, see + * get_grouping_expression(). + */ + gvis = list_copy(root->grouped_var_list); + foreach(lc, root->grouped_var_list) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + int relid = -1; + + /* Only interested in grouping expressions. */ + if (IsA(gvi->gvexpr, Aggref)) + continue; + + while ((relid = bms_next_member(rel->relids, relid)) >= 0) + { + GroupedVarInfo *gvi_trans; + + gvi_trans = translate_expression_to_rels(root, gvi, relid); + if (gvi_trans != NULL) + gvis = lappend(gvis, gvi_trans); + } + } + + /* + * Check if some aggregates or grouping expressions can be evaluated in + * this relation's target, and collect all vars referenced by these + * aggregates / grouping expressions; + */ + found_other_rel_agg = false; + foreach(lc, gvis) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + + /* + * The subset includes gv_eval_at uninitialized, which includes + * Aggref.aggstar. + */ + if (bms_is_subset(gvi->gv_eval_at, rel->relids)) + { + /* + * init_grouping_targets will handle plain Var grouping + * expressions because it needs to look them up in + * grouped_var_list anyway. + */ + if (IsA(gvi->gvexpr, Var)) + continue; + + /* + * Currently, GroupedVarInfo only handles Vars and Aggrefs. + */ + Assert(IsA(gvi->gvexpr, Aggref)); + + /* We only derive grouped expressions using ECs, not aggregates */ + Assert(!gvi->derived); + + gvi->agg_partial = (Aggref *) copyObject(gvi->gvexpr); + mark_partial_aggref(gvi->agg_partial, AGGSPLIT_INITIAL_SERIAL); + + /* + * Accept the aggregate. + */ + aggregates = lappend(aggregates, gvi); + } + else if (IsA(gvi->gvexpr, Aggref)) + { + /* + * Remember that there is at least one aggregate expression that + * needs something else than this rel. + */ + found_other_rel_agg = true; + + /* + * This condition effectively terminates creation of the + * RelAggInfo, so there's no reason to check the next + * GroupedVarInfo. + */ + break; + } + } + + /* + * Grouping makes little sense w/o aggregate function and w/o grouping + * expressions. + */ + if (aggregates == NIL) + { + list_free(gvis); + return NULL; + } + + /* + * Give up if some other aggregate(s) need relations other than the + * current one. + * + * If the aggregate needs the current rel plus anything else, then the + * problem is that grouping of the current relation could make some input + * variables unavailable for the "higher aggregate", and it'd also + * decrease the number of input rows the "higher aggregate" receives. + * + * If the aggregate does not even need the current rel, then neither the + * current rel nor anything else should be grouped because we do not + * support join of two grouped relations. + */ + if (found_other_rel_agg) + { + list_free(gvis); + return NULL; + } + + /* + * Create target for grouped paths as well as one for the input paths of + * the aggregation paths. + */ + target = create_empty_pathtarget(); + agg_input = create_empty_pathtarget(); + + /* + * Cannot suitable targets for the aggregation push-down be derived? + */ + if (!init_grouping_targets(root, rel, target, agg_input, gvis, + &grp_exprs_extra)) + { + list_free(gvis); + return NULL; + } + + list_free(gvis); + + /* + * Aggregation push-down makes no sense w/o grouping expressions. + */ + if ((list_length(target->exprs) + list_length(grp_exprs_extra)) == 0) + return NULL; + + group_clauses_final = root->parse->groupClause; + + /* + * If the aggregation target should have extra grouping expressions (in + * order to emit input vars for join conditions), add them now. This step + * includes assignment of tleSortGroupRef's which we can generate now. + */ + if (list_length(grp_exprs_extra) > 0) + { + Index sortgroupref; + + /* + * We'll have to add some clauses, but query group clause must be + * preserved. + */ + group_clauses_final = list_copy(group_clauses_final); + + /* + * Always start at root->max_sortgroupref. The extra grouping + * expressions aren't used during the final aggregation, so the + * sortgroupref values don't need to be unique across the query. Thus + * we don't have to increase root->max_sortgroupref, which makes + * recognition of the extra grouping expressions pretty easy. + */ + sortgroupref = root->max_sortgroupref; + + /* + * Generate the SortGroupClause's and add the expressions to the + * target. + */ + foreach(lc, grp_exprs_extra) + { + Var *var = lfirst_node(Var, lc); + SortGroupClause *cl = makeNode(SortGroupClause); + + /* + * Initialize the SortGroupClause. + * + * As the final aggregation will not use this grouping expression, + * we don't care whether sortop is < or >. The value of + * nulls_first should not matter for the same reason. + */ + cl->tleSortGroupRef = ++sortgroupref; + get_sort_group_operators(var->vartype, + false, true, false, + &cl->sortop, &cl->eqop, NULL, + &cl->hashable); + group_clauses_final = lappend(group_clauses_final, cl); + add_column_to_pathtarget(target, (Expr *) var, + cl->tleSortGroupRef); + + /* + * The aggregation input target must emit this var too. + */ + add_column_to_pathtarget(agg_input, (Expr *) var, + cl->tleSortGroupRef); + } + } + + /* + * Add aggregates to the grouping target. + */ + add_aggregates_to_target(root, target, aggregates); + + /* + * Build a list of grouping expressions and a list of the corresponding + * SortGroupClauses. + */ + i = 0; + foreach(lc, target->exprs) + { + Index sortgroupref = 0; + SortGroupClause *cl; + Expr *texpr; + + texpr = (Expr *) lfirst(lc); + + if (IsA(texpr, Aggref)) + { + /* + * Once we see Aggref, no grouping expressions should follow. + */ + break; + } + + /* + * Find the clause by sortgroupref. + */ + sortgroupref = target->sortgrouprefs[i++]; + + /* + * Besides being an aggregate, the target expression should have no + * other reason then being a column of a relation functionally + * dependent on the GROUP BY clause. So it's not actually a grouping + * column. + */ + if (sortgroupref == 0) + continue; + + /* + * group_clause_final contains the "local" clauses, so this search + * should succeed. + */ + cl = get_sortgroupref_clause(sortgroupref, group_clauses_final); + + result->group_clauses = list_append_unique(result->group_clauses, + cl); + + /* + * Add only unique clauses because of joins (both sides of a join can + * point at the same grouping clause). XXX Is it worth adding a bool + * argument indicating that we're dealing with join right now? + */ + result->group_exprs = list_append_unique(result->group_exprs, + texpr); + } + + /* + * Since neither target nor agg_input is supposed to be identical to the + * source reltarget, compute the width and cost again. + * + * target does not yet contain aggregates, but these will be accounted by + * AggPath. + */ + set_pathtarget_cost_width(root, target); + set_pathtarget_cost_width(root, agg_input); + + result->relids = bms_copy(rel->relids); + result->target = target; + result->agg_input = agg_input; + + /* Finally collect the aggregates. */ + while (lc != NULL) + { + Aggref *aggref = lfirst_node(Aggref, lc); + + /* + * Partial aggregation is what the grouped paths should do. + */ + result->agg_exprs = lappend(result->agg_exprs, aggref); + lc = lnext(target->exprs, lc); + } + + /* The "input_rows" field should be set by caller. */ + return result; +} + +/* + * Initialize target for grouped paths (target) as well as a target for paths + * that generate input for aggregation (agg_input). + * + * group_exprs_extra_p receives a list of Var nodes for which we need to + * construct SortGroupClause. Those vars will then be used as additional + * grouping expressions, for the sake of join clauses. + * + * gvis a list of GroupedVarInfo's possibly useful for rel. + * + * Return true iff the targets could be initialized. + */ +static bool +init_grouping_targets(PlannerInfo *root, RelOptInfo *rel, + PathTarget *target, PathTarget *agg_input, + List *gvis, List **group_exprs_extra_p) +{ + ListCell *lc1, + *lc2; + List *unresolved = NIL; + List *unresolved_sortgrouprefs = NIL; + + foreach(lc1, rel->reltarget->exprs) + { + Var *tvar; + bool is_grouping; + Index sortgroupref = 0; + bool derived = false; + bool needed_by_aggregate; + + /* + * Given that PlaceHolderVar currently prevents us from doing + * aggregation push-down, the source target cannot contain anything + * more complex than a Var. + */ + tvar = lfirst_node(Var, lc1); + + is_grouping = is_grouping_expression(gvis, (Expr *) tvar, + &sortgroupref, &derived); + + /* + * Derived grouping expressions should not be referenced by the query + * targetlist, so let them fall into vars_unresolved. It'll be checked + * later if the current targetlist needs them. For example, we should + * not automatically use Var as a grouping expression if the only + * reason for it to be in the plain relation target is that it's + * referenced by aggregate argument, and it happens to be in the same + * EC as any grouping expression. + */ + if (is_grouping && !derived) + { + Assert(sortgroupref > 0); + + /* + * It's o.k. to use the target expression for grouping. + */ + add_column_to_pathtarget(target, (Expr *) tvar, sortgroupref); + + /* + * As for agg_input, add the original expression but set + * sortgroupref in addition. + */ + add_column_to_pathtarget(agg_input, (Expr *) tvar, sortgroupref); + + /* Process the next expression. */ + continue; + } + + /* + * Is this Var needed in the query targetlist for anything else than + * aggregate input? + */ + needed_by_aggregate = false; + foreach(lc2, root->grouped_var_list) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc2); + ListCell *lc3; + List *vars; + + if (!IsA(gvi->gvexpr, Aggref)) + continue; + + if (!bms_is_member(tvar->varno, gvi->gv_eval_at)) + continue; + + /* + * XXX Consider some sort of caching. + */ + vars = pull_var_clause((Node *) gvi->gvexpr, PVC_RECURSE_AGGREGATES); + foreach(lc3, vars) + { + Var *var = lfirst_node(Var, lc3); + + if (equal(var, tvar)) + { + needed_by_aggregate = true; + break; + } + } + list_free(vars); + if (needed_by_aggregate) + break; + } + + if (needed_by_aggregate) + { + bool found = false; + + foreach(lc2, root->processed_tlist) + { + TargetEntry *te = lfirst_node(TargetEntry, lc2); + + if (IsA(te->expr, Aggref)) + continue; + + if (equal(te->expr, tvar)) + { + found = true; + break; + } + } + + /* + * If it's only Aggref input, add it to the aggregation input + * target and that's it. + */ + if (!found) + { + add_new_column_to_pathtarget(agg_input, (Expr *) tvar); + continue; + } + } + + /* + * Further investigation involves dependency check, for which we need + * to have all the (plain-var) grouping expressions gathered. + */ + unresolved = lappend(unresolved, tvar); + unresolved_sortgrouprefs = lappend_int(unresolved_sortgrouprefs, + sortgroupref); + } + + /* + * Check for other possible reasons for the var to be in the plain target. + */ + forboth(lc1, unresolved, lc2, unresolved_sortgrouprefs) + { + Var *var = lfirst_node(Var, lc1); + Index sortgroupref = lfirst_int(lc2); + RangeTblEntry *rte; + List *deps = NIL; + Relids relids_subtract; + int ndx; + RelOptInfo *baserel; + + rte = root->simple_rte_array[var->varno]; + + /* + * Check if the Var can be in the grouping key even though it's not + * mentioned by the GROUP BY clause (and could not be derived using + * ECs). + */ + if (sortgroupref == 0 && + check_functional_grouping(rte->relid, var->varno, + var->varlevelsup, + target->exprs, &deps)) + { + /* + * The var shouldn't be actually used as a grouping key (instead, + * the one this depends on will be), so sortgroupref should not be + * important. + */ + add_new_column_to_pathtarget(target, (Expr *) var); + add_new_column_to_pathtarget(agg_input, (Expr *) var); + + /* + * The var may or may not be present in generic grouping + * expression(s) in addition, but this is handled elsewhere. + */ + continue; + } + + /* + * Isn't the expression needed by joins above the current rel? + * + * The relids we're not interested in do include 0, which is the + * top-level targetlist. The only reason for relids to contain 0 + * should be that arg_var is referenced either by aggregate or by + * grouping expression, but right now we're interested in the *other* + * reasons. (As soon aggregation is pushed down, the aggregates in the + * query targetlist no longer need direct reference to arg_var + * anyway.) + */ + relids_subtract = bms_copy(rel->relids); + bms_add_member(relids_subtract, 0); + + baserel = find_base_rel(root, var->varno); + ndx = var->varattno - baserel->min_attr; + if (bms_nonempty_difference(baserel->attr_needed[ndx], + relids_subtract)) + { + /* + * The variable is needed by a join involving this relation. That + * case includes variable that is referenced by a generic grouping + * expression. + * + * The only way to bring this var to the aggregation output is to + * add it to the grouping expressions too. + */ + if (sortgroupref > 0) + { + /* + * The var could be recognized as a potentially useful + * grouping expression at the top of the loop, so we can add + * it to the grouping target, as well as to the agg_input. + */ + add_column_to_pathtarget(target, (Expr *) var, sortgroupref); + add_column_to_pathtarget(agg_input, (Expr *) var, sortgroupref); + } + else + { + /* + * Since root->parse->groupClause is not supposed to contain + * this expression, we need to construct special + * SortGroupClause. Its tleSortGroupRef needs to be unique + * within target_agg, so postpone creation of the + * SortGroupRefs until we're done with the iteration of + * rel->reltarget->exprs. + */ + *group_exprs_extra_p = lappend(*group_exprs_extra_p, var); + } + } + else + { + /* + * As long as the query is semantically correct, arriving here + * means that the var is referenced by a generic grouping + * expression but not referenced by any join. + * + * create_rel_agg_info() should add this variable to "agg_input" + * target and also add the whole generic expression to "target", + * but that's subject to future enhancement of the aggregate + * push-down feature. + */ + return false; + } + } + + return true; +} diff --git a/src/backend/optimizer/util/tlist.c b/src/backend/optimizer/util/tlist.c index 7ccb10e4e1..01dc70b27f 100644 --- a/src/backend/optimizer/util/tlist.c +++ b/src/backend/optimizer/util/tlist.c @@ -827,6 +827,62 @@ apply_pathtarget_labeling_to_tlist(List *tlist, PathTarget *target) } /* + * For each aggregate grouping expression or aggregate to the grouped target. + * + * Caller passes the expressions in the form of GroupedVarInfos so that we + * don't have to look for gvid. + */ +void +add_aggregates_to_target(PlannerInfo *root, PathTarget *target, + List *expressions) +{ + ListCell *lc; + + /* Create the vars and add them to the target. */ + foreach(lc, expressions) + { + GroupedVarInfo *gvi; + + gvi = lfirst_node(GroupedVarInfo, lc); + add_column_to_pathtarget(target, (Expr *) gvi->agg_partial, + gvi->sortgroupref); + } +} + +/* + * Find out if expr can be used as grouping expression in reltarget. + * + * sortgroupref and is_derived reflect the ->sortgroupref and ->derived fields + * of the corresponding GroupedVarInfo. + */ +bool +is_grouping_expression(List *gvis, Expr *expr, Index *sortgroupref, + bool *is_derived) +{ + ListCell *lc; + + foreach(lc, gvis) + { + GroupedVarInfo *gvi = lfirst_node(GroupedVarInfo, lc); + + if (IsA(gvi->gvexpr, Aggref)) + continue; + + if (equal(gvi->gvexpr, expr)) + { + Assert(gvi->sortgroupref > 0); + + *sortgroupref = gvi->sortgroupref; + *is_derived = gvi->derived; + return true; + } + } + + /* The expression cannot be used as grouping key. */ + return false; +} + +/* * split_pathtarget_at_srfs * Split given PathTarget into multiple levels to position SRFs safely * diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index fc463601ff..8c1929c758 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1022,6 +1022,15 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, { + {"enable_agg_pushdown", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("Enables aggregation push-down."), + NULL + }, + &enable_agg_pushdown, + false, + NULL, NULL, NULL + }, + { {"enable_parallel_append", PGC_USERSET, QUERY_TUNING_METHOD, gettext_noop("Enables the planner's use of parallel append plans."), NULL, diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 11027cdb10..25d058ef2a 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -224,6 +224,7 @@ typedef enum NodeTag T_IndexOptInfo, T_ForeignKeyOptInfo, T_ParamPathInfo, + T_RelAggInfo, T_Path, T_IndexPath, T_BitmapHeapPath, @@ -268,6 +269,7 @@ typedef enum NodeTag T_SpecialJoinInfo, T_AppendRelInfo, T_PlaceHolderInfo, + T_GroupedVarInfo, T_MinMaxAggInfo, T_PlannerParamItem, T_RollupData, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 38dc186623..988d6220c8 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -241,6 +241,23 @@ struct PlannerInfo struct RelInfoList *join_rel_list; /* list of join-relation RelOptInfos */ /* + * grouped_rel_list is a list of RelOptInfos that represent grouped + * relations, both base relations and joins. Unlike join_rel_list, base + * relations are accepted because grouped base relation is of rather + * limited scope, i.e. it's only needed during join search. Thus it does + * not deserve separate storage like simple_rel_array. + */ + struct RelInfoList *grouped_rel_list; /* list of grouped relation + * RelOptInfos */ + + /* + * agg_info_list contains one instance of RelAggInfo per an item of + * grouped_rel_list. + */ + struct RelInfoList *agg_info_list; /* list of grouped relation + * RelAggInfos */ + + /* * When doing a dynamic-programming-style join search, join_rel_level[k] * is a list of all join-relation RelOptInfos of level k, and * join_cur_level is the current level. New join-relation RelOptInfos are @@ -285,6 +302,8 @@ struct PlannerInfo List *placeholder_list; /* list of PlaceHolderInfos */ + List *grouped_var_list; /* List of GroupedVarInfos. */ + List *fkey_list; /* list of ForeignKeyOptInfos */ List *query_pathkeys; /* desired pathkeys for query_planner() */ @@ -316,6 +335,12 @@ struct PlannerInfo */ List *processed_tlist; + /* + * The maximum ressortgroupref among target entries in processed_list. + * Useful when adding extra grouping expressions for partial aggregation. + */ + int max_sortgroupref; + /* Fields filled during create_plan() for use in setrefs.c */ AttrNumber *grouping_map; /* for GroupingFunc fixup */ List *minmax_aggs; /* List of MinMaxAggInfos */ @@ -755,6 +780,60 @@ typedef struct RelInfoList } RelInfoList; /* + * RelAggInfo + * Information needed to create grouped paths for base rels and joins. + * + * "relids" is the set of base-relation identifiers, just like with + * RelOptInfo. + * + * "target" will be used as pathtarget if partial aggregation is applied to + * base relation or join. The same target will also --- if the relation is a + * join --- be used to joinin grouped path to a non-grouped one. This target + * can contain plain-Var grouping expressions and Aggref nodes. + * + * Note: There's a convention that Aggref expressions are supposed to follow + * the other expressions of the target. Iterations of ->exprs may rely on this + * arrangement. + * + * "agg_input" contains Vars used either as grouping expressions or aggregate + * arguments. Paths providing the aggregation plan with input data should use + * this target. The only difference from reltarget of the non-grouped relation + * is that some items can have sortgroupref initialized. + * + * "input_rows" is the estimated number of input rows for AggPath. It's + * actually just a workspace for users of the structure, i.e. not initialized + * when instance of the structure is created. + * + * "group_clauses" and "group_exprs" are lists of SortGroupClause and the + * corresponding grouping expressions respectively. + * + * "agg_exprs" is a list of Aggref nodes for the aggregation of the relation's + * paths. + * + * "rel_grouped" is the relation containing the partially aggregated paths. + */ +typedef struct RelAggInfo +{ + NodeTag type; + + Relids relids; /* Base rels contained in this grouped rel. */ + + struct PathTarget *target; /* Target for grouped paths. */ + + struct PathTarget *agg_input; /* pathtarget of paths that generate input + * for aggregation paths. */ + + double input_rows; + + List *group_clauses; + List *group_exprs; + + List *agg_exprs; /* Aggref expressions. */ + + RelOptInfo *rel_grouped; /* Grouped relation. */ +} RelAggInfo; + +/* * IndexOptInfo * Per-index information for planning/optimization * @@ -2273,6 +2352,26 @@ typedef struct PlaceHolderInfo } PlaceHolderInfo; /* + * GroupedVarInfo exists for each expression that can be used as an aggregate + * or grouping expression evaluated below a join. + */ +typedef struct GroupedVarInfo +{ + NodeTag type; + + Expr *gvexpr; /* the represented expression. */ + Aggref *agg_partial; /* if gvexpr is aggregate, agg_partial is the + * corresponding partial aggregate */ + Index sortgroupref; /* If gvexpr is a grouping expression, this is + * the tleSortGroupRef of the corresponding + * SortGroupClause. */ + Relids gv_eval_at; /* lowest level we can evaluate the expression + * at or NULL if it can happen anywhere. */ + bool derived; /* derived from another GroupedVarInfo using + * equeivalence classes? */ +} GroupedVarInfo; + +/* * This struct describes one potentially index-optimizable MIN/MAX aggregate * function. MinMaxAggPath contains a list of these, and if we accept that * path, the list is stored into root->minmax_aggs for use during setrefs.c. diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index 2f9aeec4a7..90f6995dc2 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -55,4 +55,6 @@ extern void CommuteOpExpr(OpExpr *clause); extern Query *inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte); +extern GroupedVarInfo *translate_expression_to_rels(PlannerInfo *root, + GroupedVarInfo *gvi, Index relid); #endif /* CLAUSES_H */ diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 182ffeef4b..2d05ff5a15 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -210,6 +210,14 @@ extern AggPath *create_agg_path(PlannerInfo *root, List *qual, const AggClauseCosts *aggcosts, double numGroups); +extern AggPath *create_agg_sorted_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + RelAggInfo *agg_info); +extern AggPath *create_agg_hashed_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + RelAggInfo *agg_info); extern GroupingSetsPath *create_groupingsets_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, @@ -281,14 +289,21 @@ extern void setup_append_rel_array(PlannerInfo *root); extern void expand_planner_arrays(PlannerInfo *root, int add_size); extern RelOptInfo *build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent); +extern RelOptInfo *build_simple_grouped_rel(PlannerInfo *root, int relid, + RelAggInfo **agg_info_p); extern RelOptInfo *find_base_rel(PlannerInfo *root, int relid); extern RelOptInfo *find_join_rel(PlannerInfo *root, Relids relids); +extern void add_grouped_rel(PlannerInfo *root, RelOptInfo *rel, + RelAggInfo *agg_info); +extern RelOptInfo *find_grouped_rel(PlannerInfo *root, Relids relids, + RelAggInfo **agg_info_p); extern RelOptInfo *build_join_rel(PlannerInfo *root, Relids joinrelids, RelOptInfo *outer_rel, RelOptInfo *inner_rel, SpecialJoinInfo *sjinfo, - List **restrictlist_ptr); + List **restrictlist_ptr, + RelAggInfo *agg_info); extern Relids min_join_parameterization(PlannerInfo *root, Relids joinrelids, RelOptInfo *outer_rel, @@ -314,5 +329,5 @@ extern RelOptInfo *build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, RelOptInfo *inner_rel, RelOptInfo *parent_joinrel, List *restrictlist, SpecialJoinInfo *sjinfo, JoinType jointype); - +extern RelAggInfo *create_rel_agg_info(PlannerInfo *root, RelOptInfo *rel); #endif /* PATHNODE_H */ diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 7345137d1d..3006ebf5aa 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -21,6 +21,7 @@ * allpaths.c */ extern PGDLLIMPORT bool enable_geqo; +extern PGDLLIMPORT bool enable_agg_pushdown; extern PGDLLIMPORT int geqo_threshold; extern PGDLLIMPORT int min_parallel_table_scan_size; extern PGDLLIMPORT int min_parallel_index_scan_size; @@ -52,8 +53,14 @@ extern RelOptInfo *make_one_rel(PlannerInfo *root, List *joinlist); extern RelOptInfo *standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels); + extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel, bool override_rows); +extern void generate_grouping_paths(PlannerInfo *root, + RelOptInfo *rel_grouped, + RelOptInfo *rel_plain, + RelAggInfo *agg_info); + extern int compute_parallel_worker(RelOptInfo *rel, double heap_pages, double index_pages, int max_workers); extern void create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index e7aaddd50d..761f610a96 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -28,7 +28,8 @@ typedef void (*query_pathkeys_callback) (PlannerInfo *root, void *extra); * prototypes for plan/planmain.c */ extern RelOptInfo *query_planner(PlannerInfo *root, - query_pathkeys_callback qp_callback, void *qp_extra); + query_pathkeys_callback qp_callback, void *qp_extra, + RelOptInfo **final_rel_grouped_p); /* * prototypes for plan/planagg.c @@ -69,6 +70,7 @@ extern void add_other_rels_to_query(PlannerInfo *root); extern void build_base_rel_tlists(PlannerInfo *root, List *final_tlist); extern void add_vars_to_targetlist(PlannerInfo *root, List *vars, Relids where_needed, bool create_new_ph); +extern void setup_aggregate_pushdown(PlannerInfo *root); extern void find_lateral_references(PlannerInfo *root); extern void create_lateral_join_info(PlannerInfo *root); extern List *deconstruct_jointree(PlannerInfo *root); diff --git a/src/include/optimizer/tlist.h b/src/include/optimizer/tlist.h index 8a2378f968..e48f37bac2 100644 --- a/src/include/optimizer/tlist.h +++ b/src/include/optimizer/tlist.h @@ -50,8 +50,14 @@ extern void split_pathtarget_at_srfs(PlannerInfo *root, PathTarget *target, PathTarget *input_target, List **targets, List **targets_contain_srfs); +/* TODO Find the best location (position and in some cases even file) for the + * following ones. */ +extern void add_aggregates_to_target(PlannerInfo *root, PathTarget *target, + List *expressions); +extern bool is_grouping_expression(List *gvis, Expr *expr, + Index *sortgroupref, bool *is_derived); + /* Convenience macro to get a PathTarget with valid cost/width fields */ #define create_pathtarget(root, tlist) \ set_pathtarget_cost_width(root, make_pathtarget_from_tlist(tlist)) - #endif /* TLIST_H */ diff --git a/src/test/regress/expected/agg_pushdown.out b/src/test/regress/expected/agg_pushdown.out new file mode 100644 index 0000000000..b3a97f86d6 --- /dev/null +++ b/src/test/regress/expected/agg_pushdown.out @@ -0,0 +1,217 @@ +BEGIN; +CREATE TABLE agg_pushdown_parent ( + i int primary key, + x int); +CREATE TABLE agg_pushdown_child1 ( + j int, + parent int references agg_pushdown_parent, + v double precision, + PRIMARY KEY (j, parent)); +CREATE INDEX ON agg_pushdown_child1(parent); +CREATE TABLE agg_pushdown_child2 ( + k int, + parent int references agg_pushdown_parent, + v double precision, + PRIMARY KEY (k, parent));; +INSERT INTO agg_pushdown_parent(i, x) +SELECT n, n +FROM generate_series(0, 7) AS s(n); +INSERT INTO agg_pushdown_child1(j, parent, v) +SELECT 128 * i + n, i, random() +FROM generate_series(0, 127) AS s(n), agg_pushdown_parent; +INSERT INTO agg_pushdown_child2(k, parent, v) +SELECT 128 * i + n, i, random() +FROM generate_series(0, 127) AS s(n), agg_pushdown_parent; +COMMIT; +ANALYZE; +SET enable_agg_pushdown TO on; +SET enable_nestloop TO on; +SET enable_hashjoin TO off; +SET enable_mergejoin TO off; +-- Perform scan of a table, aggregate the result, join it to the other table +-- and finalize the aggregation. +-- +-- In addition, check that functionally dependent column "c.x" can be +-- referenced by SELECT although GROUP BY references "p.i". +EXPLAIN (COSTS off) +SELECT p.x, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + QUERY PLAN +-------------------------------------------------------------------------------------- + Finalize GroupAggregate + Group Key: p.i + -> Sort + Sort Key: p.i + -> Nested Loop + -> Partial HashAggregate + Group Key: c1.parent + -> Seq Scan on agg_pushdown_child1 c1 + -> Index Scan using agg_pushdown_parent_pkey on agg_pushdown_parent p + Index Cond: (i = c1.parent) +(10 rows) + +-- The same for hash join. +SET enable_nestloop TO off; +SET enable_hashjoin TO on; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + QUERY PLAN +------------------------------------------------------------------ + Finalize GroupAggregate + Group Key: p.i + -> Sort + Sort Key: p.i + -> Hash Join + Hash Cond: (p.i = c1.parent) + -> Seq Scan on agg_pushdown_parent p + -> Hash + -> Partial HashAggregate + Group Key: c1.parent + -> Seq Scan on agg_pushdown_child1 c1 +(11 rows) + +-- The same for merge join. +SET enable_hashjoin TO off; +SET enable_mergejoin TO on; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + QUERY PLAN +------------------------------------------------------------ + Finalize GroupAggregate + Group Key: p.i + -> Merge Join + Merge Cond: (p.i = c1.parent) + -> Sort + Sort Key: p.i + -> Seq Scan on agg_pushdown_parent p + -> Sort + Sort Key: c1.parent + -> Partial HashAggregate + Group Key: c1.parent + -> Seq Scan on agg_pushdown_child1 c1 +(12 rows) + +SET enable_nestloop TO on; +SET enable_hashjoin TO on; +-- Scan index on agg_pushdown_child1(parent) column and aggregate the result +-- using AGG_SORTED strategy. +SET enable_seqscan TO off; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Finalize GroupAggregate + Group Key: p.i + -> Nested Loop + -> Partial GroupAggregate + Group Key: c1.parent + -> Index Scan using agg_pushdown_child1_parent_idx on agg_pushdown_child1 c1 + -> Index Only Scan using agg_pushdown_parent_pkey on agg_pushdown_parent p + Index Cond: (i = c1.parent) +(8 rows) + +SET enable_seqscan TO on; +-- Join "c1" to "p.x" column, i.e. one that is not in the GROUP BY clause. The +-- planner should still use "c1.parent" as grouping expression for partial +-- aggregation, although it's not in the same equivalence class as the GROUP +-- BY expression ("p.i"). The reason to use "c1.parent" for partial +-- aggregation is that this is the only way for "c1" to provide the join +-- expression with input data. +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.x GROUP BY p.i; + QUERY PLAN +------------------------------------------------------------------ + Finalize GroupAggregate + Group Key: p.i + -> Sort + Sort Key: p.i + -> Hash Join + Hash Cond: (p.x = c1.parent) + -> Seq Scan on agg_pushdown_parent p + -> Hash + -> Partial HashAggregate + Group Key: c1.parent + -> Seq Scan on agg_pushdown_child1 c1 +(11 rows) + +-- Perform nestloop join between agg_pushdown_child1 and agg_pushdown_child2 +-- and aggregate the result. +SET enable_nestloop TO on; +SET enable_hashjoin TO off; +SET enable_mergejoin TO off; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Finalize GroupAggregate + Group Key: p.i + -> Sort + Sort Key: p.i + -> Nested Loop + -> Partial HashAggregate + Group Key: c1.parent + -> Nested Loop + -> Seq Scan on agg_pushdown_child1 c1 + -> Index Scan using agg_pushdown_child2_pkey on agg_pushdown_child2 c2 + Index Cond: ((k = c1.j) AND (parent = c1.parent)) + -> Index Only Scan using agg_pushdown_parent_pkey on agg_pushdown_parent p + Index Cond: (i = c1.parent) +(13 rows) + +-- The same for hash join. +SET enable_nestloop TO off; +SET enable_hashjoin TO on; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; + QUERY PLAN +---------------------------------------------------------------------------------------- + Finalize GroupAggregate + Group Key: p.i + -> Sort + Sort Key: p.i + -> Hash Join + Hash Cond: (p.i = c1.parent) + -> Seq Scan on agg_pushdown_parent p + -> Hash + -> Partial HashAggregate + Group Key: c1.parent + -> Hash Join + Hash Cond: ((c1.parent = c2.parent) AND (c1.j = c2.k)) + -> Seq Scan on agg_pushdown_child1 c1 + -> Hash + -> Seq Scan on agg_pushdown_child2 c2 +(15 rows) + +-- The same for merge join. +SET enable_hashjoin TO off; +SET enable_mergejoin TO on; +SET enable_seqscan TO off; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Finalize GroupAggregate + Group Key: p.i + -> Merge Join + Merge Cond: (c1.parent = p.i) + -> Sort + Sort Key: c1.parent + -> Partial HashAggregate + Group Key: c1.parent + -> Merge Join + Merge Cond: ((c1.j = c2.k) AND (c1.parent = c2.parent)) + -> Index Scan using agg_pushdown_child1_pkey on agg_pushdown_child1 c1 + -> Index Scan using agg_pushdown_child2_pkey on agg_pushdown_child2 c2 + -> Index Only Scan using agg_pushdown_parent_pkey on agg_pushdown_parent p +(13 rows) + diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index a1c90eb905..55cfd3fff7 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -72,6 +72,7 @@ select count(*) >= 0 as ok from pg_prepared_xacts; select name, setting from pg_settings where name like 'enable%'; name | setting --------------------------------+--------- + enable_agg_pushdown | off enable_bitmapscan | on enable_gathermerge | on enable_hashagg | on @@ -89,7 +90,7 @@ select name, setting from pg_settings where name like 'enable%'; enable_seqscan | on enable_sort | on enable_tidscan | on -(17 rows) +(18 rows) -- Test that the pg_timezone_names and pg_timezone_abbrevs views are -- more-or-less working. We can't test their contents in any great detail diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index a39ca1012a..4f0ba643db 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -140,6 +140,7 @@ test: amutils test: stats_ext test: select_parallel test: write_parallel +test: agg_pushdown test: publication test: subscription test: select_views diff --git a/src/test/regress/sql/agg_pushdown.sql b/src/test/regress/sql/agg_pushdown.sql new file mode 100644 index 0000000000..6cf2ed9c20 --- /dev/null +++ b/src/test/regress/sql/agg_pushdown.sql @@ -0,0 +1,117 @@ +BEGIN; + +CREATE TABLE agg_pushdown_parent ( + i int primary key, + x int); + +CREATE TABLE agg_pushdown_child1 ( + j int, + parent int references agg_pushdown_parent, + v double precision, + PRIMARY KEY (j, parent)); + +CREATE INDEX ON agg_pushdown_child1(parent); + +CREATE TABLE agg_pushdown_child2 ( + k int, + parent int references agg_pushdown_parent, + v double precision, + PRIMARY KEY (k, parent));; + +INSERT INTO agg_pushdown_parent(i, x) +SELECT n, n +FROM generate_series(0, 7) AS s(n); + +INSERT INTO agg_pushdown_child1(j, parent, v) +SELECT 128 * i + n, i, random() +FROM generate_series(0, 127) AS s(n), agg_pushdown_parent; + +INSERT INTO agg_pushdown_child2(k, parent, v) +SELECT 128 * i + n, i, random() +FROM generate_series(0, 127) AS s(n), agg_pushdown_parent; + +COMMIT; +ANALYZE; + +SET enable_agg_pushdown TO on; + +SET enable_nestloop TO on; +SET enable_hashjoin TO off; +SET enable_mergejoin TO off; + +-- Perform scan of a table, aggregate the result, join it to the other table +-- and finalize the aggregation. +-- +-- In addition, check that functionally dependent column "c.x" can be +-- referenced by SELECT although GROUP BY references "p.i". +EXPLAIN (COSTS off) +SELECT p.x, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + +-- The same for hash join. +SET enable_nestloop TO off; +SET enable_hashjoin TO on; + +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + +-- The same for merge join. +SET enable_hashjoin TO off; +SET enable_mergejoin TO on; + +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + +SET enable_nestloop TO on; +SET enable_hashjoin TO on; + +-- Scan index on agg_pushdown_child1(parent) column and aggregate the result +-- using AGG_SORTED strategy. +SET enable_seqscan TO off; +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.i GROUP BY p.i; + +SET enable_seqscan TO on; + +-- Join "c1" to "p.x" column, i.e. one that is not in the GROUP BY clause. The +-- planner should still use "c1.parent" as grouping expression for partial +-- aggregation, although it's not in the same equivalence class as the GROUP +-- BY expression ("p.i"). The reason to use "c1.parent" for partial +-- aggregation is that this is the only way for "c1" to provide the join +-- expression with input data. +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v) FROM agg_pushdown_parent AS p JOIN agg_pushdown_child1 +AS c1 ON c1.parent = p.x GROUP BY p.i; + +-- Perform nestloop join between agg_pushdown_child1 and agg_pushdown_child2 +-- and aggregate the result. +SET enable_nestloop TO on; +SET enable_hashjoin TO off; +SET enable_mergejoin TO off; + +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; + +-- The same for hash join. +SET enable_nestloop TO off; +SET enable_hashjoin TO on; + +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; + +-- The same for merge join. +SET enable_hashjoin TO off; +SET enable_mergejoin TO on; +SET enable_seqscan TO off; + +EXPLAIN (COSTS off) +SELECT p.i, avg(c1.v + c2.v) FROM agg_pushdown_parent AS p JOIN +agg_pushdown_child1 AS c1 ON c1.parent = p.i JOIN agg_pushdown_child2 AS c2 ON +c2.parent = p.i WHERE c1.j = c2.k GROUP BY p.i; -- 2.13.7 --=-=-=-- ^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v8 1/1] add to_binary() and to_oct() @ 2023-07-25 23:09 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: Nathan Bossart @ 2023-07-25 23:09 UTC (permalink / raw) --- doc/src/sgml/func.sgml | 47 ++++++++++++++- src/backend/utils/adt/varlena.c | 86 +++++++++++++++++++-------- src/include/catalog/pg_proc.dat | 12 ++++ src/test/regress/expected/strings.out | 62 ++++++++++++++++++- src/test/regress/sql/strings.sql | 15 ++++- 5 files changed, 192 insertions(+), 30 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index be2f54c914..835c08905e 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -3737,6 +3737,50 @@ repeat('Pg', 4) <returnvalue>PgPgPgPg</returnvalue> </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>to_binary</primary> + </indexterm> + <function>to_binary</function> ( <type>integer</type> ) + <returnvalue>text</returnvalue> + </para> + <para role="func_signature"> + <function>to_binary</function> ( <type>bigint</type> ) + <returnvalue>text</returnvalue> + </para> + <para> + Converts the number to its equivalent two's complement binary + representation. + </para> + <para> + <literal>to_binary(2147483647)</literal> + <returnvalue>1111111111111111111111111111111</returnvalue> + </para></entry> + </row> + + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>to_oct</primary> + </indexterm> + <function>to_oct</function> ( <type>integer</type> ) + <returnvalue>text</returnvalue> + </para> + <para role="func_signature"> + <function>to_oct</function> ( <type>bigint</type> ) + <returnvalue>text</returnvalue> + </para> + <para> + Converts the number to its equivalent two's complement octal + representation. + </para> + <para> + <literal>to_oct(2147483647)</literal> + <returnvalue>17777777777</returnvalue> + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -3750,7 +3794,8 @@ repeat('Pg', 4) <returnvalue>PgPgPgPg</returnvalue> <returnvalue>text</returnvalue> </para> <para> - Converts the number to its equivalent hexadecimal representation. + Converts the number to its equivalent two's complement hexadecimal + representation. </para> <para> <literal>to_hex(2147483647)</literal> diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index b1ec5c32ce..cac3577480 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4919,53 +4919,87 @@ array_to_text_internal(FunctionCallInfo fcinfo, ArrayType *v, return result; } -#define HEXBASE 16 /* - * Convert an int32 to a string containing a base 16 (hex) representation of - * the number. + * Workhorse for to_binary, to_oct, and to_hex. Note that base must be > 1 and + * <= 16. */ -Datum -to_hex32(PG_FUNCTION_ARGS) +static inline text * +convert_to_base(uint64 value, int base) { - uint32 value = (uint32) PG_GETARG_INT32(0); - char *ptr; const char *digits = "0123456789abcdef"; - char buf[32]; /* bigger than needed, but reasonable */ - ptr = buf + sizeof(buf) - 1; - *ptr = '\0'; + /* We size the buffer for to_binary's longest possible return value. */ + char buf[sizeof(uint64) * BITS_PER_BYTE]; + char *const end = buf + sizeof(buf); + char *ptr = end; + + Assert(base > 1); + Assert(base <= 16); do { - *--ptr = digits[value % HEXBASE]; - value /= HEXBASE; + *--ptr = digits[value % base]; + value /= base; } while (ptr > buf && value); - PG_RETURN_TEXT_P(cstring_to_text(ptr)); + return cstring_to_text_with_len(ptr, end - ptr); +} + +/* + * Convert an integer to a string containing a base-2 (binary) representation + * of the number. + */ +Datum +to_binary32(PG_FUNCTION_ARGS) +{ + uint64 value = (uint32) PG_GETARG_INT32(0); + + PG_RETURN_TEXT_P(convert_to_base(value, 2)); +} +Datum +to_binary64(PG_FUNCTION_ARGS) +{ + uint64 value = (uint64) PG_GETARG_INT64(0); + + PG_RETURN_TEXT_P(convert_to_base(value, 2)); } /* - * Convert an int64 to a string containing a base 16 (hex) representation of + * Convert an integer to a string containing a base-8 (oct) representation of * the number. */ Datum -to_hex64(PG_FUNCTION_ARGS) +to_oct32(PG_FUNCTION_ARGS) +{ + uint64 value = (uint32) PG_GETARG_INT32(0); + + PG_RETURN_TEXT_P(convert_to_base(value, 8)); +} +Datum +to_oct64(PG_FUNCTION_ARGS) { uint64 value = (uint64) PG_GETARG_INT64(0); - char *ptr; - const char *digits = "0123456789abcdef"; - char buf[32]; /* bigger than needed, but reasonable */ - ptr = buf + sizeof(buf) - 1; - *ptr = '\0'; + PG_RETURN_TEXT_P(convert_to_base(value, 8)); +} - do - { - *--ptr = digits[value % HEXBASE]; - value /= HEXBASE; - } while (ptr > buf && value); +/* + * Convert an integer to a string containing a base-16 (hex) representation of + * the number. + */ +Datum +to_hex32(PG_FUNCTION_ARGS) +{ + uint64 value = (uint32) PG_GETARG_INT32(0); + + PG_RETURN_TEXT_P(convert_to_base(value, 16)); +} +Datum +to_hex64(PG_FUNCTION_ARGS) +{ + uint64 value = (uint64) PG_GETARG_INT64(0); - PG_RETURN_TEXT_P(cstring_to_text(ptr)); + PG_RETURN_TEXT_P(convert_to_base(value, 16)); } /* diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 12fac15ceb..b653351605 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -3707,6 +3707,18 @@ { oid => '2768', descr => 'split string by pattern', proname => 'regexp_split_to_array', prorettype => '_text', proargtypes => 'text text text', prosrc => 'regexp_split_to_array' }, +{ oid => '9030', descr => 'convert int4 number to binary', + proname => 'to_binary', prorettype => 'text', proargtypes => 'int4', + prosrc => 'to_binary32' }, +{ oid => '9031', descr => 'convert int8 number to binary', + proname => 'to_binary', prorettype => 'text', proargtypes => 'int8', + prosrc => 'to_binary64' }, +{ oid => '9032', descr => 'convert int4 number to oct', + proname => 'to_oct', prorettype => 'text', proargtypes => 'int4', + prosrc => 'to_oct32' }, +{ oid => '9033', descr => 'convert int8 number to oct', + proname => 'to_oct', prorettype => 'text', proargtypes => 'int8', + prosrc => 'to_oct64' }, { oid => '2089', descr => 'convert int4 number to hex', proname => 'to_hex', prorettype => 'text', proargtypes => 'int4', prosrc => 'to_hex32' }, diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out index 62698569e1..d24e6fa0cd 100644 --- a/src/test/regress/expected/strings.out +++ b/src/test/regress/expected/strings.out @@ -2129,8 +2129,68 @@ select split_part('@joeuser@mydatabase@','@',-2) AS "mydatabase"; (1 row) -- --- test to_hex +-- test to_binary, to_oct, and to_hex -- +select to_binary(-1234) AS "11111111111111111111101100101110"; + 11111111111111111111101100101110 +---------------------------------- + 11111111111111111111101100101110 +(1 row) + +select to_binary(-1234::bigint); + to_binary +------------------------------------------------------------------ + 1111111111111111111111111111111111111111111111111111101100101110 +(1 row) + +select to_binary(256*256*256 - 1) AS "111111111111111111111111"; + 111111111111111111111111 +-------------------------- + 111111111111111111111111 +(1 row) + +select to_binary(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "11111111111111111111111111111111"; + 11111111111111111111111111111111 +---------------------------------- + 11111111111111111111111111111111 +(1 row) + +select to_oct(-1234) AS "37777775456"; + 37777775456 +------------- + 37777775456 +(1 row) + +select to_oct(-1234::bigint) AS "1777777777777777775456"; + 1777777777777777775456 +------------------------ + 1777777777777777775456 +(1 row) + +select to_oct(256*256*256 - 1) AS "77777777"; + 77777777 +---------- + 77777777 +(1 row) + +select to_oct(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "37777777777"; + 37777777777 +------------- + 37777777777 +(1 row) + +select to_hex(-1234) AS "fffffb2e"; + fffffb2e +---------- + fffffb2e +(1 row) + +select to_hex(-1234::bigint) AS "fffffffffffffb2e"; + fffffffffffffb2e +------------------ + fffffffffffffb2e +(1 row) + select to_hex(256*256*256 - 1) AS "ffffff"; ffffff -------- diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql index ca32f6bba5..437713fdc6 100644 --- a/src/test/regress/sql/strings.sql +++ b/src/test/regress/sql/strings.sql @@ -685,10 +685,21 @@ select split_part('joeuser@mydatabase','@',-3) AS "empty string"; select split_part('@joeuser@mydatabase@','@',-2) AS "mydatabase"; -- --- test to_hex +-- test to_binary, to_oct, and to_hex -- -select to_hex(256*256*256 - 1) AS "ffffff"; +select to_binary(-1234) AS "11111111111111111111101100101110"; +select to_binary(-1234::bigint); +select to_binary(256*256*256 - 1) AS "111111111111111111111111"; +select to_binary(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "11111111111111111111111111111111"; + +select to_oct(-1234) AS "37777775456"; +select to_oct(-1234::bigint) AS "1777777777777777775456"; +select to_oct(256*256*256 - 1) AS "77777777"; +select to_oct(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "37777777777"; +select to_hex(-1234) AS "fffffb2e"; +select to_hex(-1234::bigint) AS "fffffffffffffb2e"; +select to_hex(256*256*256 - 1) AS "ffffff"; select to_hex(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "ffffffff"; -- -- 2.25.1 --ibTvN161/egqYuK8-- ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-10-27 15:13 Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-10-27 15:13 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 10/27/23 11:56 AM, shveta malik wrote: > On Wed, Oct 25, 2023 at 3:15 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> Hi, >> >> On 10/25/23 5:00 AM, shveta malik wrote: >>> On Tue, Oct 24, 2023 at 11:54 AM Drouvot, Bertrand >>> <[email protected]> wrote: >>>> >>>> Hi, >>>> >>>> On 10/23/23 2:56 PM, shveta malik wrote: >>>>> On Mon, Oct 23, 2023 at 5:52 PM Drouvot, Bertrand >>>>> <[email protected]> wrote: >>>> >>>>>> We are waiting for DEFAULT_NAPTIME_PER_CYCLE (3 minutes) before checking if there >>>>>> is new synced slot(s) to be created on the standby. Do we want to keep this behavior >>>>>> for V1? >>>>>> >>>>> >>>>> I think for the slotsync workers case, we should reduce the naptime in >>>>> the launcher to say 30sec and retain the default one of 3mins for >>>>> subscription apply workers. Thoughts? >>>>> >>>> >>>> Another option could be to keep DEFAULT_NAPTIME_PER_CYCLE and create a new >>>> API on the standby that would refresh the list of sync slot at wish, thoughts? >>>> >>> >>> Do you mean API to refresh list of DBIDs rather than sync-slots? >>> As per current design, launcher gets DBID lists for all the failover >>> slots from the primary at intervals of DEFAULT_NAPTIME_PER_CYCLE. >> >> I mean an API to get a newly created slot on the primary being created/synced on >> the standby at wish. >> >> Also let's imagine this scenario: >> >> - create logical_slot1 on the primary (and don't start using it) >> >> Then on the standby we'll get things like: >> >> 2023-10-25 08:33:36.897 UTC [740298] LOG: waiting for remote slot "logical_slot1" LSN (0/C00316A0) and catalog xmin (752) to pass local slot LSN (0/C0049530) and and catalog xmin (754) >> >> That's expected and due to the fact that ReplicationSlotReserveWal() does set the slot >> restart_lsn to a value < at the corresponding restart_lsn slot on the primary. >> >> - create logical_slot2 on the primary (and start using it) >> >> Then logical_slot2 won't be created/synced on the standby until there is activity on logical_slot1 on the primary >> that would produce things like: >> 2023-10-25 08:41:35.508 UTC [740298] LOG: wait over for remote slot "logical_slot1" as its LSN (0/C005FFD8) and catalog xmin (756) has now passed local slot LSN (0/C0049530) and catalog xmin (754) > > > Slight correction to above. As soon as we start activity on > logical_slot2, it will impact all the slots on primary, as the WALs > are consumed by all the slots. So even if there is activity on > logical_slot2, logical_slot1 creation on standby will be unblocked and > it will then move to logical_slot2 creation. eg: > > --on standby: > 2023-10-27 15:15:46.069 IST [696884] LOG: waiting for remote slot > "mysubnew1_1" LSN (0/3C97970) and catalog xmin (756) to pass local > slot LSN (0/3C979A8) and and catalog xmin (756) > > on primary: > newdb1=# select now(); > now > ---------------------------------- > 2023-10-27 15:15:51.504835+05:30 > (1 row) > > --activity on mysubnew1_3 > newdb1=# insert into tab1_3 values(1); > INSERT 0 1 > newdb1=# select now(); > now > ---------------------------------- > 2023-10-27 15:15:54.651406+05:30 > > > --on standby, mysubnew1_1 is unblocked. > 2023-10-27 15:15:56.223 IST [696884] LOG: wait over for remote slot > "mysubnew1_1" as its LSN (0/3C97A18) and catalog xmin (757) has now > passed local slot LSN (0/3C979A8) and catalog xmin (756) > > My Setup: > mysubnew1_1 -->mypubnew1_1 -->tab1_1 > mysubnew1_3 -->mypubnew1_3-->tab1_3 > Agree with your test case, but in my case I was not using pub/sub. I was not clear, so when I said: >> - create logical_slot1 on the primary (and don't start using it) I meant don't start decoding from it (like using pg_recvlogical() or pg_logical_slot_get_changes()). By using pub/sub the "don't start using it" is not satisfied. My test case is: " SELECT * FROM pg_create_logical_replication_slot('logical_slot1', 'test_decoding', false, true, true); SELECT * FROM pg_create_logical_replication_slot('logical_slot2', 'test_decoding', false, true, true); pg_recvlogical -d postgres -S logical_slot2 --no-loop --start -f - " Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-10-31 09:37 shveta malik <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: shveta malik @ 2023-10-31 09:37 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Fri, Oct 27, 2023 at 8:43 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 10/27/23 11:56 AM, shveta malik wrote: > > On Wed, Oct 25, 2023 at 3:15 PM Drouvot, Bertrand > > <[email protected]> wrote: > >> > >> Hi, > >> > >> On 10/25/23 5:00 AM, shveta malik wrote: > >>> On Tue, Oct 24, 2023 at 11:54 AM Drouvot, Bertrand > >>> <[email protected]> wrote: > >>>> > >>>> Hi, > >>>> > >>>> On 10/23/23 2:56 PM, shveta malik wrote: > >>>>> On Mon, Oct 23, 2023 at 5:52 PM Drouvot, Bertrand > >>>>> <[email protected]> wrote: > >>>> > >>>>>> We are waiting for DEFAULT_NAPTIME_PER_CYCLE (3 minutes) before checking if there > >>>>>> is new synced slot(s) to be created on the standby. Do we want to keep this behavior > >>>>>> for V1? > >>>>>> > >>>>> > >>>>> I think for the slotsync workers case, we should reduce the naptime in > >>>>> the launcher to say 30sec and retain the default one of 3mins for > >>>>> subscription apply workers. Thoughts? > >>>>> > >>>> > >>>> Another option could be to keep DEFAULT_NAPTIME_PER_CYCLE and create a new > >>>> API on the standby that would refresh the list of sync slot at wish, thoughts? > >>>> > >>> > >>> Do you mean API to refresh list of DBIDs rather than sync-slots? > >>> As per current design, launcher gets DBID lists for all the failover > >>> slots from the primary at intervals of DEFAULT_NAPTIME_PER_CYCLE. > >> > >> I mean an API to get a newly created slot on the primary being created/synced on > >> the standby at wish. > >> > >> Also let's imagine this scenario: > >> > >> - create logical_slot1 on the primary (and don't start using it) > >> > >> Then on the standby we'll get things like: > >> > >> 2023-10-25 08:33:36.897 UTC [740298] LOG: waiting for remote slot "logical_slot1" LSN (0/C00316A0) and catalog xmin (752) to pass local slot LSN (0/C0049530) and and catalog xmin (754) > >> > >> That's expected and due to the fact that ReplicationSlotReserveWal() does set the slot > >> restart_lsn to a value < at the corresponding restart_lsn slot on the primary. > >> > >> - create logical_slot2 on the primary (and start using it) > >> > >> Then logical_slot2 won't be created/synced on the standby until there is activity on logical_slot1 on the primary > >> that would produce things like: > >> 2023-10-25 08:41:35.508 UTC [740298] LOG: wait over for remote slot "logical_slot1" as its LSN (0/C005FFD8) and catalog xmin (756) has now passed local slot LSN (0/C0049530) and catalog xmin (754) > > > > > > Slight correction to above. As soon as we start activity on > > logical_slot2, it will impact all the slots on primary, as the WALs > > are consumed by all the slots. So even if there is activity on > > logical_slot2, logical_slot1 creation on standby will be unblocked and > > it will then move to logical_slot2 creation. eg: > > > > --on standby: > > 2023-10-27 15:15:46.069 IST [696884] LOG: waiting for remote slot > > "mysubnew1_1" LSN (0/3C97970) and catalog xmin (756) to pass local > > slot LSN (0/3C979A8) and and catalog xmin (756) > > > > on primary: > > newdb1=# select now(); > > now > > ---------------------------------- > > 2023-10-27 15:15:51.504835+05:30 > > (1 row) > > > > --activity on mysubnew1_3 > > newdb1=# insert into tab1_3 values(1); > > INSERT 0 1 > > newdb1=# select now(); > > now > > ---------------------------------- > > 2023-10-27 15:15:54.651406+05:30 > > > > > > --on standby, mysubnew1_1 is unblocked. > > 2023-10-27 15:15:56.223 IST [696884] LOG: wait over for remote slot > > "mysubnew1_1" as its LSN (0/3C97A18) and catalog xmin (757) has now > > passed local slot LSN (0/3C979A8) and catalog xmin (756) > > > > My Setup: > > mysubnew1_1 -->mypubnew1_1 -->tab1_1 > > mysubnew1_3 -->mypubnew1_3-->tab1_3 > > > > Agree with your test case, but in my case I was not using pub/sub. > > I was not clear, so when I said: > > >> - create logical_slot1 on the primary (and don't start using it) > > I meant don't start decoding from it (like using pg_recvlogical() or > pg_logical_slot_get_changes()). > > By using pub/sub the "don't start using it" is not satisfied. > > My test case is: > > " > SELECT * FROM pg_create_logical_replication_slot('logical_slot1', 'test_decoding', false, true, true); > SELECT * FROM pg_create_logical_replication_slot('logical_slot2', 'test_decoding', false, true, true); > pg_recvlogical -d postgres -S logical_slot2 --no-loop --start -f - > " > Okay, I am able to reproduce it now. Thanks for clarification. I have tried to change the algorithm as per suggestion by Amit in [1] [1]: https://www.postgresql.org/message-id/CAA4eK1KBL0110gamQfc62X%3D5JV8-Qjd0dw0Mq0o07cq6kE%2Bq%3Dg%40ma... This is not full proof solution but optimization over first one. Now in any sync-cycle, we take 2 attempts for slots-creation (if any slots are available to be created). In first attempt, we do not wait indefinitely on inactive slots, we wait only for a fixed amount of time and if remote-slot is still behind, then we add that to the pending list and move to the next slot. Once we are done with first attempt, in second attempt, we go for the pending ones and now we wait on each of them until the primary catches up. > Regards, > > -- > Bertrand Drouvot > PostgreSQL Contributors Team > RDS Open Source Databases > Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-07 10:21 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-07 10:21 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 10/31/23 10:37 AM, shveta malik wrote: > On Fri, Oct 27, 2023 at 8:43 PM Drouvot, Bertrand >> >> Agree with your test case, but in my case I was not using pub/sub. >> >> I was not clear, so when I said: >> >>>> - create logical_slot1 on the primary (and don't start using it) >> >> I meant don't start decoding from it (like using pg_recvlogical() or >> pg_logical_slot_get_changes()). >> >> By using pub/sub the "don't start using it" is not satisfied. >> >> My test case is: >> >> " >> SELECT * FROM pg_create_logical_replication_slot('logical_slot1', 'test_decoding', false, true, true); >> SELECT * FROM pg_create_logical_replication_slot('logical_slot2', 'test_decoding', false, true, true); >> pg_recvlogical -d postgres -S logical_slot2 --no-loop --start -f - >> " >> > > Okay, I am able to reproduce it now. Thanks for clarification. I have > tried to change the algorithm as per suggestion by Amit in [1] > > [1]: https://www.postgresql.org/message-id/CAA4eK1KBL0110gamQfc62X%3D5JV8-Qjd0dw0Mq0o07cq6kE%2Bq%3Dg%40ma... Thanks! > > This is not full proof solution but optimization over first one. Now > in any sync-cycle, we take 2 attempts for slots-creation (if any slots > are available to be created). In first attempt, we do not wait > indefinitely on inactive slots, we wait only for a fixed amount of > time and if remote-slot is still behind, then we add that to the > pending list and move to the next slot. Once we are done with first > attempt, in second attempt, we go for the pending ones and now we wait > on each of them until the primary catches up. Aren't we "just" postponing the "issue"? I mean if there is really no activity on, say, the first created slot, then once we move to the second attempt then any newly created slot from that time would wait to be synced forever, no? Looking at V30: + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotPersist(); + + ereport(LOG, errmsg("created slot \"%s\" locally", remote_slot->name)); I think this message is confusing as the slot has been created before it, here: + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); So that it shows up in pg_replication_slots before this message is emitted (and that specially true/worst for non active slots). Maybe something like "newly locally created slot XXX has been synced..."? While at it, would that make sense to move + slot->data.failover = true; once we stop waiting for this slot? I think that would avoid confusion if one query pg_replication_slots while we are still waiting for this slot to be synced, thoughts? (currently we can see pg_replication_slots.synced_slot set to true while we are still waiting). Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-07 10:55 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Amit Kapila @ 2023-11-07 10:55 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Tue, Nov 7, 2023 at 3:51 PM Drouvot, Bertrand <[email protected]> wrote: > > On 10/31/23 10:37 AM, shveta malik wrote: > > On Fri, Oct 27, 2023 at 8:43 PM Drouvot, Bertrand > >> > >> Agree with your test case, but in my case I was not using pub/sub. > >> > >> I was not clear, so when I said: > >> > >>>> - create logical_slot1 on the primary (and don't start using it) > >> > >> I meant don't start decoding from it (like using pg_recvlogical() or > >> pg_logical_slot_get_changes()). > >> > >> By using pub/sub the "don't start using it" is not satisfied. > >> > >> My test case is: > >> > >> " > >> SELECT * FROM pg_create_logical_replication_slot('logical_slot1', 'test_decoding', false, true, true); > >> SELECT * FROM pg_create_logical_replication_slot('logical_slot2', 'test_decoding', false, true, true); > >> pg_recvlogical -d postgres -S logical_slot2 --no-loop --start -f - > >> " > >> > > > > Okay, I am able to reproduce it now. Thanks for clarification. I have > > tried to change the algorithm as per suggestion by Amit in [1] > > > > [1]: https://www.postgresql.org/message-id/CAA4eK1KBL0110gamQfc62X%3D5JV8-Qjd0dw0Mq0o07cq6kE%2Bq%3Dg%40ma... > > Thanks! > > > > > This is not full proof solution but optimization over first one. Now > > in any sync-cycle, we take 2 attempts for slots-creation (if any slots > > are available to be created). In first attempt, we do not wait > > indefinitely on inactive slots, we wait only for a fixed amount of > > time and if remote-slot is still behind, then we add that to the > > pending list and move to the next slot. Once we are done with first > > attempt, in second attempt, we go for the pending ones and now we wait > > on each of them until the primary catches up. > > Aren't we "just" postponing the "issue"? I mean if there is really no activity > on, say, the first created slot, then once we move to the second attempt then any newly > created slot from that time would wait to be synced forever, no? > We have to wait at some point in time for such inactive slots and the same is true even for manually created slots on standby. Do you have any better ideas to deal with it? > Looking at V30: > > + /* Update lsns of slot to remote slot's current position */ > + local_slot_update(remote_slot); > + ReplicationSlotPersist(); > + > + ereport(LOG, errmsg("created slot \"%s\" locally", remote_slot->name)); > > I think this message is confusing as the slot has been created before it, here: > > + else > + { > + TransactionId xmin_horizon = InvalidTransactionId; > + ReplicationSlot *slot; > + > + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > + remote_slot->two_phase, false); > > So that it shows up in pg_replication_slots before this message is emitted (and that > specially true/worst for non active slots). > > Maybe something like "newly locally created slot XXX has been synced..."? > > While at it, would that make sense to move > > + slot->data.failover = true; > > once we stop waiting for this slot? I think that would avoid confusion if one > query pg_replication_slots while we are still waiting for this slot to be synced, > thoughts? (currently we can see pg_replication_slots.synced_slot set to true > while we are still waiting). > The failover property of the slot is different from whether the slot has been synced yet, so we can't change the location of marking it but we can try to improve when to show that slot has been synced. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-07 14:28 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-07 14:28 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/7/23 11:55 AM, Amit Kapila wrote: > On Tue, Nov 7, 2023 at 3:51 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> On 10/31/23 10:37 AM, shveta malik wrote: >>> On Fri, Oct 27, 2023 at 8:43 PM Drouvot, Bertrand >>>> >>>> Agree with your test case, but in my case I was not using pub/sub. >>>> >>>> I was not clear, so when I said: >>>> >>>>>> - create logical_slot1 on the primary (and don't start using it) >>>> >>>> I meant don't start decoding from it (like using pg_recvlogical() or >>>> pg_logical_slot_get_changes()). >>>> >>>> By using pub/sub the "don't start using it" is not satisfied. >>>> >>>> My test case is: >>>> >>>> " >>>> SELECT * FROM pg_create_logical_replication_slot('logical_slot1', 'test_decoding', false, true, true); >>>> SELECT * FROM pg_create_logical_replication_slot('logical_slot2', 'test_decoding', false, true, true); >>>> pg_recvlogical -d postgres -S logical_slot2 --no-loop --start -f - >>>> " >>>> >>> >>> Okay, I am able to reproduce it now. Thanks for clarification. I have >>> tried to change the algorithm as per suggestion by Amit in [1] >>> >>> [1]: https://www.postgresql.org/message-id/CAA4eK1KBL0110gamQfc62X%3D5JV8-Qjd0dw0Mq0o07cq6kE%2Bq%3Dg%40ma... >> >> Thanks! >> >>> >>> This is not full proof solution but optimization over first one. Now >>> in any sync-cycle, we take 2 attempts for slots-creation (if any slots >>> are available to be created). In first attempt, we do not wait >>> indefinitely on inactive slots, we wait only for a fixed amount of >>> time and if remote-slot is still behind, then we add that to the >>> pending list and move to the next slot. Once we are done with first >>> attempt, in second attempt, we go for the pending ones and now we wait >>> on each of them until the primary catches up. >> >> Aren't we "just" postponing the "issue"? I mean if there is really no activity >> on, say, the first created slot, then once we move to the second attempt then any newly >> created slot from that time would wait to be synced forever, no? >> > > We have to wait at some point in time for such inactive slots and the > same is true even for manually created slots on standby. Do you have > any better ideas to deal with it? > What about: - get rid of the second attempt and the pending_slot_list - keep the wait_count and PrimaryCatchupWaitAttempt logic so basically, get rid of: /* * Now sync the pending slots which were failed to be created in first * attempt. */ foreach(cell, pending_slot_list) { RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); /* Wait until the primary server catches up */ PrimaryCatchupWaitAttempt = 0; synchronize_one_slot(wrconn, remote_slot, NULL); } and the pending_slot_list list. That way, for each slot that have not been created and synced yet: - it will be created on the standby - we will wait up to PrimaryCatchupWaitAttempt attempts - the slot will be synced or removed on/from the standby That way an inactive slot on the primary would not "block" any other slots on the standby. By "created" here I mean calling ReplicationSlotCreate() (not to be confused with emitting "ereport(LOG, errmsg("created slot \"%s\" locally", remote_slot->name)); " which is confusing as mentioned up-thread). The problem I can see with this proposal is that the "sync" window waiting for slot activity on the primary is "only" during the PrimaryCatchupWaitAttempt attempts (as the slot will be dropped/recreated). If we think this window is too short we could: - increase it or - don't drop the slot once created (even if there is no activity on the primary during PrimaryCatchupWaitAttempt attempts) so that the next loop of attempts will compare with "older" LSN/xmin (as compare to dropping and re-creating the slot). That way the window would be since the initial slot creation. Thoughts? Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 03:50 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Amit Kapila @ 2023-11-08 03:50 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Tue, Nov 7, 2023 at 7:58 PM Drouvot, Bertrand <[email protected]> wrote: > > On 11/7/23 11:55 AM, Amit Kapila wrote: > >>> > >>> This is not full proof solution but optimization over first one. Now > >>> in any sync-cycle, we take 2 attempts for slots-creation (if any slots > >>> are available to be created). In first attempt, we do not wait > >>> indefinitely on inactive slots, we wait only for a fixed amount of > >>> time and if remote-slot is still behind, then we add that to the > >>> pending list and move to the next slot. Once we are done with first > >>> attempt, in second attempt, we go for the pending ones and now we wait > >>> on each of them until the primary catches up. > >> > >> Aren't we "just" postponing the "issue"? I mean if there is really no activity > >> on, say, the first created slot, then once we move to the second attempt then any newly > >> created slot from that time would wait to be synced forever, no? > >> > > > > We have to wait at some point in time for such inactive slots and the > > same is true even for manually created slots on standby. Do you have > > any better ideas to deal with it? > > > > What about: > > - get rid of the second attempt and the pending_slot_list > - keep the wait_count and PrimaryCatchupWaitAttempt logic > > so basically, get rid of: > > /* > * Now sync the pending slots which were failed to be created in first > * attempt. > */ > foreach(cell, pending_slot_list) > { > RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); > > /* Wait until the primary server catches up */ > PrimaryCatchupWaitAttempt = 0; > > synchronize_one_slot(wrconn, remote_slot, NULL); > } > > and the pending_slot_list list. > > That way, for each slot that have not been created and synced yet: > > - it will be created on the standby > - we will wait up to PrimaryCatchupWaitAttempt attempts > - the slot will be synced or removed on/from the standby > > That way an inactive slot on the primary would not "block" > any other slots on the standby. > > By "created" here I mean calling ReplicationSlotCreate() (not to be confused > with emitting "ereport(LOG, errmsg("created slot \"%s\" locally", remote_slot->name)); " > which is confusing as mentioned up-thread). > > The problem I can see with this proposal is that the "sync" window waiting > for slot activity on the primary is "only" during the PrimaryCatchupWaitAttempt > attempts (as the slot will be dropped/recreated). > > If we think this window is too short we could: > > - increase it > or > - don't drop the slot once created (even if there is no activity > on the primary during PrimaryCatchupWaitAttempt attempts) so that > the next loop of attempts will compare with "older" LSN/xmin (as compare to > dropping and re-creating the slot). That way the window would be since the > initial slot creation. > Yeah, this sounds reasonable but we can't mark such slots to be synced/available for use after failover. I think if we want to follow this approach then we need to also monitor these slots for any change in the consecutive cycles and if we are able to sync them then accordingly we enable them to use after failover. Another somewhat related point is that right now, we just wait for the change on the first slot (the patch refers to it as the monitoring slot) for computing nap_time before which we will recheck all the slots. I think we can improve that as well such that even if any slot's information is changed, we don't consider changing naptime. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 07:02 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-08 07:02 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/8/23 4:50 AM, Amit Kapila wrote: > On Tue, Nov 7, 2023 at 7:58 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> If we think this window is too short we could: >> >> - increase it >> or >> - don't drop the slot once created (even if there is no activity >> on the primary during PrimaryCatchupWaitAttempt attempts) so that >> the next loop of attempts will compare with "older" LSN/xmin (as compare to >> dropping and re-creating the slot). That way the window would be since the >> initial slot creation. >> > > Yeah, this sounds reasonable but we can't mark such slots to be > synced/available for use after failover. Yeah, currently we are fine as slots are dropped in wait_for_primary_slot_catchup() if we are not in recovery anymore. > I think if we want to follow > this approach then we need to also monitor these slots for any change > in the consecutive cycles and if we are able to sync them then > accordingly we enable them to use after failover. What about to add a new field in ReplicationSlotPersistentData indicating that we are waiting for "sync" and drop such slots during promotion and /or if not in recovery? > Another somewhat related point is that right now, we just wait for the > change on the first slot (the patch refers to it as the monitoring > slot) for computing nap_time before which we will recheck all the > slots. I think we can improve that as well such that even if any > slot's information is changed, we don't consider changing naptime. > Yeah, that sounds reasonable to me. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 08:57 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Amit Kapila @ 2023-11-08 08:57 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Wed, Nov 8, 2023 at 12:32 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 11/8/23 4:50 AM, Amit Kapila wrote: > > > I think if we want to follow > > this approach then we need to also monitor these slots for any change > > in the consecutive cycles and if we are able to sync them then > > accordingly we enable them to use after failover. > > What about to add a new field in ReplicationSlotPersistentData > indicating that we are waiting for "sync" and drop such slots during promotion and > /or if not in recovery? > This patch is already adding 'synced' flag in ReplicationSlotPersistentData to distinguish synced slots so that we can disallow decoding on then in standby and disallow to drop those. I suggest we change that field to have multiple states where one of the states would indicate that the initial sync of the slot is done. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 09:49 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-08 09:49 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/8/23 9:57 AM, Amit Kapila wrote: > On Wed, Nov 8, 2023 at 12:32 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> Hi, >> >> On 11/8/23 4:50 AM, Amit Kapila wrote: >> >>> I think if we want to follow >>> this approach then we need to also monitor these slots for any change >>> in the consecutive cycles and if we are able to sync them then >>> accordingly we enable them to use after failover. >> >> What about to add a new field in ReplicationSlotPersistentData >> indicating that we are waiting for "sync" and drop such slots during promotion and >> /or if not in recovery? >> > > This patch is already adding 'synced' flag in > ReplicationSlotPersistentData to distinguish synced slots so that we > can disallow decoding on then in standby and disallow to drop those. I > suggest we change that field to have multiple states where one of the > states would indicate that the initial sync of the slot is done. > Yeah, agree. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 11:50 shveta malik <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: shveta malik @ 2023-11-08 11:50 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Wed, Nov 8, 2023 at 3:19 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 11/8/23 9:57 AM, Amit Kapila wrote: > > On Wed, Nov 8, 2023 at 12:32 PM Drouvot, Bertrand > > <[email protected]> wrote: > >> > >> Hi, > >> > >> On 11/8/23 4:50 AM, Amit Kapila wrote: > >> > >>> I think if we want to follow > >>> this approach then we need to also monitor these slots for any change > >>> in the consecutive cycles and if we are able to sync them then > >>> accordingly we enable them to use after failover. > >> > >> What about to add a new field in ReplicationSlotPersistentData > >> indicating that we are waiting for "sync" and drop such slots during promotion and > >> /or if not in recovery? > >> > > > > This patch is already adding 'synced' flag in > > ReplicationSlotPersistentData to distinguish synced slots so that we > > can disallow decoding on then in standby and disallow to drop those. I > > suggest we change that field to have multiple states where one of the > > states would indicate that the initial sync of the slot is done. > > > > Yeah, agree. > I am working on this implementation. This sync-state is even needed for cascading standbys to know when to start syncing the slots from the first standby. It should start syncing only after the first standby has finished initialization of it (i.e. wait for primary is over) and not before that. Unrelated to above, if there is a user slot on standby with the same name which the slot-sync worker is trying to create, then shall it emit a warning and skip the sync of that slot or shall it throw an error? thanks Shveta ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-08 14:39 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-08 14:39 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/8/23 12:50 PM, shveta malik wrote: > On Wed, Nov 8, 2023 at 3:19 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> Hi, >> >> On 11/8/23 9:57 AM, Amit Kapila wrote: >>> On Wed, Nov 8, 2023 at 12:32 PM Drouvot, Bertrand >>> <[email protected]> wrote: >>>> >>>> Hi, >>>> >>>> On 11/8/23 4:50 AM, Amit Kapila wrote: >>>> >>>>> I think if we want to follow >>>>> this approach then we need to also monitor these slots for any change >>>>> in the consecutive cycles and if we are able to sync them then >>>>> accordingly we enable them to use after failover. >>>> >>>> What about to add a new field in ReplicationSlotPersistentData >>>> indicating that we are waiting for "sync" and drop such slots during promotion and >>>> /or if not in recovery? >>>> >>> >>> This patch is already adding 'synced' flag in >>> ReplicationSlotPersistentData to distinguish synced slots so that we >>> can disallow decoding on then in standby and disallow to drop those. I >>> suggest we change that field to have multiple states where one of the >>> states would indicate that the initial sync of the slot is done. >>> >> >> Yeah, agree. >> > > I am working on this implementation. Thanks! > This sync-state is even needed > for cascading standbys to know when to start syncing the slots from > the first standby. It should start syncing only after the first > standby has finished initialization of it (i.e. wait for primary is > over) and not before that. > Yeah, makes sense. > Unrelated to above, if there is a user slot on standby with the same > name which the slot-sync worker is trying to create, then shall it > emit a warning and skip the sync of that slot or shall it throw an > error? > I'd vote for emit a warning and move on to the next slot if any. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-09 02:41 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 2 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-09 02:41 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand <[email protected]> wrote: > > > Unrelated to above, if there is a user slot on standby with the same > > name which the slot-sync worker is trying to create, then shall it > > emit a warning and skip the sync of that slot or shall it throw an > > error? > > > > I'd vote for emit a warning and move on to the next slot if any. > But then it could take time for users to know the actual problem and they probably notice it after failover. OTOH, if we throw an error then probably they will come to know earlier because the slot sync mechanism would be stopped. Do you have reasons to prefer giving a WARNING and skipping creating such slots? I expect this WARNING to keep getting repeated in LOGs because the consecutive sync tries will again generate a WARNING. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-09 03:25 Amit Kapila <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 2 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-09 03:25 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thu, Nov 9, 2023 at 8:11 AM Amit Kapila <[email protected]> wrote: > > On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand > <[email protected]> wrote: > > > > > Unrelated to above, if there is a user slot on standby with the same > > > name which the slot-sync worker is trying to create, then shall it > > > emit a warning and skip the sync of that slot or shall it throw an > > > error? > > > > > > > I'd vote for emit a warning and move on to the next slot if any. > > > > But then it could take time for users to know the actual problem and > they probably notice it after failover. OTOH, if we throw an error > then probably they will come to know earlier because the slot sync > mechanism would be stopped. Do you have reasons to prefer giving a > WARNING and skipping creating such slots? I expect this WARNING to > keep getting repeated in LOGs because the consecutive sync tries will > again generate a WARNING. > Apart from the above, I would like to discuss the slot sync work distribution strategy of this patch. The current implementation as explained in the commit message [1] works well if the slots belong to multiple databases. It is clear from the data in emails [2][3][4] that having more workers really helps if the slots belong to multiple databases. But I think if all the slots belong to one or very few databases then such a strategy won't be as good. Now, on one hand, we get very good numbers for a particular workload with the strategy used in the patch but OTOH it may not be adaptable to various different kinds of workloads. So, I have a question whether we should try to optimize this strategy for various kinds of workloads or for the first version let's use a single-slot sync-worker and then we can enhance the functionality in later patches either in PG17 itself or in PG18 or later versions. One thing to note is that a lot of the complexity of the patch is attributed to the multi-worker strategy which may still not be efficient, so there is an argument to go with a simpler single-slot sync-worker strategy and then enhance it in future versions as we learn more about various workloads. It will also help to develop this feature incrementally instead of doing all the things in one go and taking a much longer time than it should. Thoughts? [1] - "The replication launcher on the physical standby queries primary to get the list of dbids for failover logical slots. Once it gets the dbids, if dbids < max_slotsync_workers, it starts only that many workers, and if dbids > max_slotsync_workers, it starts max_slotsync_workers and divides the work equally among them. Each worker is then responsible to keep on syncing the logical slots belonging to the DBs assigned to it. Each slot-sync worker will have its own dbids list. Since the upper limit of this dbid-count is not known, it needs to be handled using dsa. We initially allocated memory to hold 100 dbids for each worker. If this limit is exhausted, we reallocate this memory with size incremented again by 100." [2] - https://www.postgresql.org/message-id/CAJpy0uD2F43avuXy_yQv7Wa3kpUwioY_Xn955xdmd6vX0ME6%3Dg%40mail.g... [3] - https://www.postgresql.org/message-id/CAFPTHDZw2G3Pax0smymMjfPqdPcZhMWo36f9F%2BTwNTs0HFxK%2Bw%40mail... [4] - https://www.postgresql.org/message-id/CAJpy0uD%3DDevMxTwFVsk_%3DxHqYNH8heptwgW6AimQ9fbRmx4ioQ%40mail... -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-09 10:54 shveta malik <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 3 replies; 55+ messages in thread From: shveta malik @ 2023-11-09 10:54 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Thu, Nov 9, 2023 at 8:56 AM Amit Kapila <[email protected]> wrote: > > On Thu, Nov 9, 2023 at 8:11 AM Amit Kapila <[email protected]> wrote: > > > > On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > > > > Unrelated to above, if there is a user slot on standby with the same > > > > name which the slot-sync worker is trying to create, then shall it > > > > emit a warning and skip the sync of that slot or shall it throw an > > > > error? > > > > > > > > > > I'd vote for emit a warning and move on to the next slot if any. > > > > > > > But then it could take time for users to know the actual problem and > > they probably notice it after failover. OTOH, if we throw an error > > then probably they will come to know earlier because the slot sync > > mechanism would be stopped. Do you have reasons to prefer giving a > > WARNING and skipping creating such slots? I expect this WARNING to > > keep getting repeated in LOGs because the consecutive sync tries will > > again generate a WARNING. > > > > Apart from the above, I would like to discuss the slot sync work > distribution strategy of this patch. The current implementation as > explained in the commit message [1] works well if the slots belong to > multiple databases. It is clear from the data in emails [2][3][4] that > having more workers really helps if the slots belong to multiple > databases. But I think if all the slots belong to one or very few > databases then such a strategy won't be as good. Now, on one hand, we > get very good numbers for a particular workload with the strategy used > in the patch but OTOH it may not be adaptable to various different > kinds of workloads. So, I have a question whether we should try to > optimize this strategy for various kinds of workloads or for the first > version let's use a single-slot sync-worker and then we can enhance > the functionality in later patches either in PG17 itself or in PG18 or > later versions. One thing to note is that a lot of the complexity of > the patch is attributed to the multi-worker strategy which may still > not be efficient, so there is an argument to go with a simpler > single-slot sync-worker strategy and then enhance it in future > versions as we learn more about various workloads. It will also help > to develop this feature incrementally instead of doing all the things > in one go and taking a much longer time than it should. > > Thoughts? > > [1] - "The replication launcher on the physical standby queries > primary to get the list of dbids for failover logical slots. Once it > gets the dbids, if dbids < max_slotsync_workers, it starts only that > many workers, and if dbids > max_slotsync_workers, it starts > max_slotsync_workers and divides the work equally among them. Each > worker is then responsible to keep on syncing the logical slots > belonging to the DBs assigned to it. > > Each slot-sync worker will have its own dbids list. Since the upper > limit of this dbid-count is not known, it needs to be handled using > dsa. We initially allocated memory to hold 100 dbids for each worker. > If this limit is exhausted, we reallocate this memory with size > incremented again by 100." > > [2] - https://www.postgresql.org/message-id/CAJpy0uD2F43avuXy_yQv7Wa3kpUwioY_Xn955xdmd6vX0ME6%3Dg%40mail.g... > [3] - https://www.postgresql.org/message-id/CAFPTHDZw2G3Pax0smymMjfPqdPcZhMWo36f9F%2BTwNTs0HFxK%2Bw%40mail... > [4] - https://www.postgresql.org/message-id/CAJpy0uD%3DDevMxTwFVsk_%3DxHqYNH8heptwgW6AimQ9fbRmx4ioQ%40mail... > > -- > With Regards, > Amit Kapila. PFA v32 patches which has below changes: 1) Changed how standby_slot_names is handled. On reanalyzing, logical decoding context might not be the best place to cache the standby slot list, because not all the callers(1. user backend. 2. walsender 3. slotsync worker) can access the logical decoding ctx. To make the access of the list consistent, cache the list in a global variable instead. Also, to avoid the trouble of allocating and freeing the list at various places, we [re]initialize the list in the GUC assign hook, it would be easier for caller to use the list. 2) Changed 'bool synced' in ReplicationSlotPersistentData to 'char sync_state'. Values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for primary to catch up. 'r': ready for periodic syncs. 3) Improved slot-creation logic in slot sync worker. Now any active slot's sync is not blocked by inactive slot's creation. The worker attempts pinging the primary server a fixed number of times and waits for it to catch-up with local-slot's lsn, after that it moves to the next slot. The worker reattempts the wait for pending ones in the next sync-cycle. Meanwhile any such slot (waiting for primary to catch-up) is not dropped but sync_status is marked as 'i'. Once the worker finishes initialization for such a slot (in any of the sync-cycles), sync_state of slot is changed to 'r'. 4) The slots with state 'i' are dropped by the slot-sync worker when it finds out that it is no longer in standby mode and then it exits. 5) Cascading standby does not sync slots with 'sync_state' = 'i' from the first standby. 6) Changed the naptime computation logic. Now during each sync-cycle, if any of the received slots is updated, we retain default-naptime else we increase the naptime provided inactivity time reaches threshold. 7) Added warning for cases where a user-slot with the same name is already present which slot-sync worker is trying to create. Sync for such slots is skipped. Changes for 1 are in patch001 and patch003. Changes for 2-7 are in patch002. Thank You Hou-san for working on 1. Open Question: 1) Currently I have put drop slot logic for slots with 'sync_state=i' in slot-sync worker. Do we need to put it somewhere in promotion-logic as well? Perhaps in WaitForWALToBecomeAvailable() where we call XLogShutdownWalRcv after checking 'CheckForStandbyTrigger'. Thoughts? thanks Shveta Attachments: [application/octet-stream] v32-0003-Allow-slot-sync-workers-to-wait-for-the-cascadin.patch (7.9K, ../../CAJpy0uC2gYybBcvYCW68wvR-3akWnFbr7x66nZg7XTyjELW9iw@mail.gmail.com/2-v32-0003-Allow-slot-sync-workers-to-wait-for-the-cascadin.patch) download | inline diff: From 875217a4083100bc0091fabe4d5fd3d07589816f Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Thu, 9 Nov 2023 10:56:51 +0530 Subject: [PATCH v32 3/3] Allow slot-sync workers to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync workers. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index f0a3f01d99..95d7ba2414 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -82,6 +82,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -536,6 +539,52 @@ drop_obsolete_slots(Oid *dbids, List *remote_slot_list) } } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Construct Slot Query * @@ -741,6 +790,7 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) long naptime = WORKER_DEFAULT_NAPTIME_MS; Oid *dbids; int count = 0; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -815,6 +865,9 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for lsns and xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -857,6 +910,17 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) ExecClearTuple(slot); } + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (list_length(remote_slot_list)) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -918,12 +982,18 @@ remote_connect() * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndSlots(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -941,7 +1011,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot-sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -957,7 +1028,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -1027,7 +1101,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index a5f5fccde9..cdf0afae13 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1678,7 +1678,7 @@ WalSndWakeupNeeded() * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndSlots(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1701,13 +1701,12 @@ WalSndRereadConfigAndSlots(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1774,10 +1773,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index ecbd3526c5..25c522993f 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndSlots(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.34.1 [application/octet-stream] v32-0002-Add-logical-slot-sync-capability-to-the-physical.patch (119.5K, ../../CAJpy0uC2gYybBcvYCW68wvR-3akWnFbr7x66nZg7XTyjELW9iw@mail.gmail.com/3-v32-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From a20ea83175e1d3e9ce08b970235effcce880f387 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v32 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker(s) on the standby server ping the primary at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. GUC 'max_slotsync_workers' defines the maximum number of slot-sync workers on the standby. This parameter can only be set at server start. The replication launcher on the physical standby queries primary to get the list of dbids for failover logical slots. Once it gets the dbids, if dbids < max_slotsync_workers, it starts only that many workers and if dbids > max_slotsync_workers, it starts max_slotsync_workers and divides the work equally among them. Each worker is then responsible to keep on syncing the logical slots belonging to the DBs assigned to it. Each slot-sync worker will have its own dbids list. Since the upper limit of this dbid-count is not known, it needs to be handled using dsa. We initially allocate memory to hold 100 dbids for each worker. If this limit is exhausted, we reallocate this memory with size incremented again by 100. The nap time of worker is tuned according to the activity on the primary. Each worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. Each worker uses one slot (first one assigned to it) for monitoring purpose. If there is no change in lsn of that slot for some threshold time, nap time is increased to 10sec and as soon as a change is observed, nap time is reduced back to 10ms. The logical slots created by slot-sync workers on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). --- doc/src/sgml/config.sgml | 56 +- doc/src/sgml/system-views.sgml | 17 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 89 ++ src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 1033 ++++++++++++++-- src/backend/replication/logical/logical.c | 12 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1063 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/repl_gram.y | 13 + src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 17 +- src/backend/replication/slotfuncs.c | 35 +- src/backend/replication/walsender.c | 76 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 25 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 8 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 30 + src/include/replication/worker_internal.h | 61 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 35 files changed, 2599 insertions(+), 148 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7136a925a9..689820b16d 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,14 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4888,50 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-max-slotsync-workers" xreflabel="max_slotsync_workers"> + <term><varname>max_slotsync_workers</varname> (<type>integer</type>) + <indexterm> + <primary><varname>max_slotsync_workers</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + Specifies maximum number of slot synchronization workers. + </para> + <para> + Slot synchronization workers are taken from the pool defined by + <varname>max_worker_processes</varname>. + </para> + <para> + The default value is 2. This parameter can only be set at server + start. + </para> + <para> + The slot-sync workers are needed for synchronization of logical replication + slots from the primary server to the physical standby so that logical + subscribers are not blocked after failover. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..cc1adf9127 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,23 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 9c595ca3c9..0bc3264d20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..d217d38641 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,8 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static List *libpqrcv_get_dbinfo_for_failover_slots(WalReceiverConn *conn); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +103,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbinfo_for_failover_slots = libpqrcv_get_dbinfo_for_failover_slots, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +418,90 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get DB info for logical failover slots + * + * It gets the DBIDs for failover logical slots from the primary server. + * The list returned by LIST_DBID_FOR_FAILOVER_SLOTS has no duplicates. + */ +static List * +libpqrcv_get_dbinfo_for_failover_slots(WalReceiverConn *conn) +{ + PGresult *res; + List *slotlist = NIL; + int ntuples; + WalRcvFailoverSlotsData *slot_data; + + res = libpqrcv_PQexec(conn->streamConn, "LIST_DBID_FOR_FAILOVER_SLOTS"); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + ereport(ERROR, + (errmsg("could not receive logical failover slots dbinfo from the primary server: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + if (PQnfields(res) != 1) + { + int nfields = PQnfields(res); + + PQclear(res); + ereport(ERROR, + (errmsg("invalid response from the primary server"), + errdetail("Could not get logical failover slots dbinfo: got %d fields, " + "expected 1", nfields))); + } + + ntuples = PQntuples(res); + for (int i = 0; i < ntuples; i++) + { + slot_data = palloc0(sizeof(WalRcvFailoverSlotsData)); + if (!PQgetisnull(res, i, 0)) + slot_data->dboid = atooid(PQgetvalue(res, i, 0)); + + slotlist = lappend(slotlist, slot_data); + } + + PQclear(res); + + return slotlist; +} + +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* If multiple dbnames are specified, then the last one will be returned */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..7247d9dcc8 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -22,6 +22,7 @@ #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "catalog/pg_authid.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -57,6 +58,22 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +int max_slotsync_workers = 2; +bool enable_syncslot = true; + +/* + * Initial allocation size for dbids array for each SlotSyncWorker in dynamic + * shared memory. + */ +#define DB_PER_WORKER_ALLOC_INIT 100 + +/* + * Once initially allocated size is exhausted for dbids array, it is extended by + * DB_PER_WORKER_ALLOC_EXTRA size. + */ +#define DB_PER_WORKER_ALLOC_EXTRA 100 + +SlotSyncWorker *MySlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +87,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorker *ss_workers; /* slot-sync workers */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +120,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorker *worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +197,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slotsync workers. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +207,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +222,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorker *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +290,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +318,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +380,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,8 +409,8 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && - TimestampDifferenceExceeds(w->launch_time, now, + if (w->hdr.in_use && !w->hdr.proc && + TimestampDifferenceExceeds(w->hdr.launch_time, now, wal_receiver_timeout)) { elog(WARNING, @@ -437,10 +466,10 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; - worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.launch_time = now; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +486,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +539,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +551,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical rep workers and slot-sync workers. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +583,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +597,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +624,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +637,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +656,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +704,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +733,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +755,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +764,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +773,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +808,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +833,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +970,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot-sync workers. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot-sync workers to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -953,6 +1001,7 @@ void ApplyLauncherShmemInit(void) { bool found; + Size ssw_size; LogicalRepCtx = (LogicalRepCtxStruct *) ShmemInitStruct("Logical Replication Launcher Data", @@ -977,6 +1026,14 @@ ApplyLauncherShmemInit(void) SpinLockInit(&worker->relmutex); } } + + /* Allocate shared-memory for slot-sync workers pool now */ + ssw_size = mul_size(max_slotsync_workers, sizeof(SlotSyncWorker)); + LogicalRepCtx->ss_workers = (SlotSyncWorker *) + ShmemInitStruct("Replication slot-sync workers", ssw_size, &found); + + if (!found) + memset(LogicalRepCtx->ss_workers, 0, ssw_size); } /* @@ -1115,13 +1172,809 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot-sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorker *worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->slot = -1; + + if (DsaPointerIsValid(worker->dbids_dp)) + { + dsa_free(worker->dbids_dsa, worker->dbids_dp); + worker->dbids_dp = InvalidDsaPointer; + } + + if (worker->dbids_dsa) + { + dsa_detach(worker->dbids_dsa); + worker->dbids_dsa = NULL; + } + + worker->dbcount = 0; + worker->last_update_time = 0; +} + +/* + * Attach Slot-sync worker to worker-slot assigned by launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach(int slot) { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + Assert(slot >= 0 && slot < max_slotsync_workers); + MySlotSyncWorker = &LogicalRepCtx->ss_workers[slot]; + MySlotSyncWorker->slot = slot; + + if (!MySlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker slot %d is " + "empty, cannot attach", slot))); + } + + if (MySlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker slot %d is " + "already used by another worker, cannot attach", slot))); + } + + MySlotSyncWorker->hdr.proc = MyProc; + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + dsa_detach((dsa_area *) DatumGetPointer(arg)); + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + MySlotSyncWorker->hdr.in_use = false; + MySlotSyncWorker->hdr.proc = NULL; + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Slot-Sync worker find. + * + * Searches the slot-sync worker pool for the worker who manages the + * specified dbid. Because a worker can manage multiple dbs, also walk + * the db array of each worker to find the match. + * + * Returns NULL if no matching worker is found. + */ +static SlotSyncWorker * +slotsync_worker_find(Oid dbid) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* Search for an attached worker for a given dbid */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *w = &LogicalRepCtx->ss_workers[widx]; + Oid *dbids; + + if (!w->hdr.in_use) + continue; + + dbids = (Oid *) dsa_get_address(w->dbids_dsa, w->dbids_dp); + for (int dbidx = 0; dbidx < w->dbcount; dbidx++) + { + if (dbids[dbidx] == dbid) + return w; + } + + } + + return NULL; +} + +/* + * Setup the slot-sync worker. + * + * DSA is used for the dbids array. Because the maximum number of dbs a + * worker can manage is not known, initially enough memory for + * DB_PER_WORKER_ALLOC_INIT dbs is allocated. If this size is exhausted, + * it can be extended using dsa free and allocate routines. + */ +static dsa_handle +slotsync_worker_setup(SlotSyncWorker *worker) +{ + dsa_area *dbids_dsa; + dsa_pointer dbids_dp; + dsa_handle dbids_dsa_handle; + MemoryContext oldcontext; + + /* Prepare the new worker. */ + worker->hdr.launch_time = GetCurrentTimestamp(); + worker->hdr.in_use = true; + + /* + * 'proc' and 'slot' will be assigned in ReplSlotSyncWorkerMain when we + * attach this worker to a particular worker-pool slot + */ + worker->hdr.proc = NULL; + worker->slot = -1; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* Ensure the memory allocated by DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + dbids_dsa = dsa_create(LWTRANCHE_SLOTSYNC_DSA); + dsa_pin(dbids_dsa); + dsa_pin_mapping(dbids_dsa); + + dbids_dp = dsa_allocate0(dbids_dsa, DB_PER_WORKER_ALLOC_INIT * sizeof(Oid)); + + /* Set-up worker */ + worker->dbcount = 0; + worker->dbids_dsa = dbids_dsa; + worker->dbids_dp = dbids_dp; + + /* Get the handle. This is the one which can be passed to worker processes */ + dbids_dsa_handle = dsa_get_handle(dbids_dsa); + + elog(DEBUG1, "allocated dsa for slot-sync worker for dbcount: %d", + DB_PER_WORKER_ALLOC_INIT); + + MemoryContextSwitchTo(oldcontext); + + return dbids_dsa_handle; +} + +/* + * Slot-sync worker launch or reuse + * + * Start new slot-sync background worker from the pool of available workers + * limited by max_slotsync_workers count. If the worker pool is exhausted, + * reuse the existing worker with minimum number of dbs. The idea is to + * always distribute the dbs equally among launched workers. + * If initially allocated dbids array is exhausted for the selected worker, + * reallocate the dbids array with increased size and copy the existing + * dbids to it and assign the new one as well. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch_or_reuse(Oid dbid) +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + uint16 generation; + SlotSyncWorker *worker = NULL; + int worker_slot = -1; + dsa_handle handle; + Oid *dbids; + bool attach; + uint32 mindbcnt = PG_UINT32_MAX; + + Assert(OidIsValid(dbid)); + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + /* + * Find unused worker slot. If all the workers are currently in use, find + * the one with minimum number of dbs and use that. + */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *w = &LogicalRepCtx->ss_workers[widx]; + + if (!w->hdr.in_use) + { + worker = w; + worker_slot = widx; + break; + } + + if (w->dbcount < mindbcnt) + { + mindbcnt = w->dbcount; + worker = w; + worker_slot = widx; + } + } + + /* + * If worker is being reused, and there is vacancy in dbids array, just + * update dbids array and dbcount and we are done. But if dbids array is + * exhausted, reallocate dbids using dsa and copy the old dbids and assign + * the new one as well. + */ + if (worker->hdr.in_use) + { + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + if (worker->dbcount < DB_PER_WORKER_ALLOC_INIT) + { + dbids[worker->dbcount++] = dbid; + } + else + { + MemoryContext oldcontext; + uint32 alloc_count; + uint32 old_dbcnt; + Oid *old_dbids; + + /* Be sure any memory allocated by DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Remember the old dbids before we reallocate dsa. */ + old_dbcnt = worker->dbcount; + old_dbids = (Oid *) palloc0(worker->dbcount * sizeof(Oid)); + memcpy(old_dbids, dbids, worker->dbcount * sizeof(Oid)); + + alloc_count = old_dbcnt + DB_PER_WORKER_ALLOC_EXTRA; + + /* Free the existing dbids and allocate new with increased size */ + if (DsaPointerIsValid(worker->dbids_dp)) + dsa_free(worker->dbids_dsa, worker->dbids_dp); + + worker->dbids_dp = dsa_allocate0(worker->dbids_dsa, + alloc_count * sizeof(Oid)); + + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + /* Copy the existing dbids */ + worker->dbcount = old_dbcnt; + memcpy(dbids, old_dbids, old_dbcnt * sizeof(Oid)); + pfree(old_dbids); + + /* Assign new dbid */ + dbids[worker->dbcount++] = dbid; + + MemoryContextSwitchTo(oldcontext); + } + + LWLockRelease(SlotSyncWorkerLock); + + ereport(LOG, + (errmsg("added database %d to replication slot-sync " + "worker %d; dbcount now: %d", + dbid, worker_slot, worker->dbcount))); + return true; + } + + /* + * Initialise the worker and setup DSA for dbids array to hold + * DB_PER_WORKER_ALLOC_INIT dbs + */ + handle = slotsync_worker_setup(worker); + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + dbids[worker->dbcount++] = dbid; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + Assert(worker_slot >= 0); + snprintf(bgw.bgw_name, BGW_MAXLEN, + "replication slot-sync worker %d", worker_slot); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot-sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + bgw.bgw_main_arg = Int32GetDatum(worker_slot); + + memcpy(bgw.bgw_extra, &handle, sizeof(dsa_handle)); + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + /* + * If attach is done, log that the worker is managing dbid, else raise a + * warning + */ + if (attach) + ereport(LOG, + (errmsg("added database %d to replication slot-sync " + "worker %d; dbcount now: %d", + dbid, worker_slot, worker->dbcount))); + else + ereport(WARNING, + (errmsg("replication slot-sync worker failed to attach to " + "worker-pool slot %d", worker_slot))); + + return attach; +} + +/* + * Internal function to stop the slot-sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorker *worker) +{ + int slot = worker->slot; + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot-sync worker %d", + slot))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Stop all the slot-sync workers in use. + */ +static void +slotsync_workers_stop() +{ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + + if (worker && worker->hdr.in_use) + slotsync_worker_stop_internal(worker); + } +} + +/* + * Drop the local 'synced' slots for all the obsoleted dbs passed in. + * + * Helper function for slotsync_remove_obsolete_dbs(). + */ +static void +slotsync_drop_obsoleted_dbs_slots(List *dbids_list) +{ + List *local_slot_list = NIL; + ListCell *lc; + Oid *dbids; + uint32 idx = 0; + uint32 dbcount = list_length(dbids_list); + + dbids = palloc0(dbcount * sizeof(Oid)); + + foreach(lc, dbids_list) + { + Oid dbid = lfirst_oid(lc); + + dbids[idx++] = dbid; + } + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* Get the list of local 'synced' slots for the given dbids. */ + local_slot_list = get_local_synced_slot_names(dbids, dbcount); + + LWLockRelease(SlotSyncWorkerLock); + + pfree(dbids); + + foreach(lc, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } +} + +/* + * Slot-sync workers remove obsolete DBs from db-list + * + * If the DBIds fetched from the primary server are lesser than the ones being + * managed by slot-sync workers, remove extra dbs from worker's db-list. This + * may happen if some logical failover slots are removed on the primary server + * or are disabled for failover. + * Also remove the local 'synced' slots belonging to such dbs. + */ +static void +slotsync_remove_obsolete_dbs(List *remote_dbs) +{ + List *removed_dbs_list = NIL; + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + /* Traverse slot-sync-workers to validate the DBs */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + Oid *dbids; + + if (!worker->hdr.in_use) + continue; + + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + for (int dbidx = 0; dbidx < worker->dbcount;) + { + Oid wdbid = dbids[dbidx]; + bool found = false; + ListCell *lc; + + /* Check if current DB is still present in remote-db-list */ + foreach(lc, remote_dbs) + { + WalRcvFailoverSlotsData *failover_slot_data = lfirst(lc); + + if (failover_slot_data->dboid == wdbid) + { + found = true; + dbidx++; + break; + } + } + + /* If not found, then delete this db from worker's db-list */ + if (!found) + { + if (dbidx < (worker->dbcount - 1)) + { + /* Shift the DBs and get rid of wdbid */ + memmove(&dbids[dbidx], &dbids[dbidx + 1], + (worker->dbcount - dbidx - 1) * sizeof(Oid)); + } + + worker->dbcount--; + + ereport(LOG, + (errmsg("removed database %d from replication slot-sync " + "worker %d; dbcount now: %d", + wdbid, worker->slot, worker->dbcount))); + + removed_dbs_list = lappend_oid(removed_dbs_list, wdbid); + } + + } + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If dbcount for any worker has become 0, shut it down. + * + * XXX: if needed in future, workers can be restarted in such a case to + * distribute the load. + */ + + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + + if (worker->hdr.in_use && !worker->dbcount) + slotsync_worker_stop_internal(worker); + } + + /* + * Drop local 'synced' slots for the dbs which are no longer eligible for + * slot-sync. + */ + if (removed_dbs_list) + { + slotsync_drop_obsoleted_dbs_slots(removed_dbs_list); + list_free(removed_dbs_list); + } +} + +/* + * Connect to the primary server for slotsync purpose and return the connection + * info. + */ +static WalReceiverConn * +slotsync_remote_connect(long *wait_time, bool *retry) +{ + WalReceiverConn *wrconn; + char *err; + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return NULL; + + if (max_slotsync_workers == 0) + return NULL; + + /* + * Since the above two GUCs are set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo) are + * compatible with slot synchronization. If not, issue warnings. + */ + + /* The primary_slot_name is not set */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return NULL; + } + + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); + return NULL; + } + + /* The dbname must be specified in primary_conninfo for slot-sync to work */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not " + "specified in primary_conninfo.")); + return NULL; + } + + wrconn = walrcv_connect(PrimaryConnInfo, false, false, + "Logical Replication Launcher", &err); + if (!wrconn) + { + ereport(ERROR, + (errmsg("could not connect to the primary server: %s for " + "slots synchronization", err))); + + /* Try connecting again in next attempt */ + *retry = true; + } + + ereport(LOG, + (errmsg("connection established to primary for slots synchronization"))); + + return wrconn; +} + +/* + * Re-read the config file. + * + * If one of the slot-sync options has changed, stop the slot-sync + * workers and disconnect the connection to primary. + */ +static void +LauncherRereadConfig(WalReceiverConn **wrconn, bool *ss_retry) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot-sync workers. The + * workers will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (feedback != hot_standby_feedback)) + { + slotsync_workers_stop(); + + if (*wrconn) + { + walrcv_disconnect(*wrconn); + *wrconn = NULL; + } + + /* Retry the connection with new GUCs */ + *ss_retry = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot-sync background workers. + * + * Connect to the primary, to get the list of DBIDs for failover logical slots. + * Then launch slot-sync workers (limited by max_slotsync_workers) where the DBs + * are distributed equally among those workers. + */ +static void +LaunchSlotSyncWorkers(long *wait_time, WalReceiverConn *wrconn) +{ + List *slots_dbs; + ListCell *lc; + MemoryContext tmpctx; + MemoryContext oldctx; + + Assert(wrconn); + + /* Use temporary context for the slot list and worker info. */ + tmpctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher slot-sync ctx", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(tmpctx); + + slots_dbs = walrcv_get_dbinfo_for_failover_slots(wrconn); + + slotsync_remove_obsolete_dbs(slots_dbs); + + foreach(lc, slots_dbs) + { + WalRcvFailoverSlotsData *slot_data = lfirst(lc); + SlotSyncWorker *w; + + Assert(OidIsValid(slot_data->dboid)); + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + w = slotsync_worker_find(slot_data->dboid); + LWLockRelease(SlotSyncWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If launch failed, adjust the wait_time to retry in the next + * sync-cycle sooner. + */ + if (!slotsync_worker_launch_or_reuse(slot_data->dboid)) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + break; + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(tmpctx); +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + bool ss_retry = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1992,30 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot-sync workers else + * launch apply workers. + */ + if (RecoveryInProgress()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); + if (wrconn == NULL && ss_retry) + wrconn = slotsync_remote_connect(&wait_time, &ss_retry); - if (w != NULL) - continue; /* worker is running already */ - - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + if (wrconn) + LaunchSlotSyncWorkers(&wait_time, wrconn); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +2030,8 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&wrconn, &ss_retry); + } /* Not reachable */ @@ -1260,7 +2062,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +2101,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..4354b906d4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -524,6 +524,18 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. + */ + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + errdetail("This slot is being synced from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..f0a3f01d99 --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1063 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot-sync workers on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot-sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS + * during a slot creation, if it still does not catch up, abort the wait. + * The ones for which wait is aborted will attempt the wait and sync in the + * next sync-cycle. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot) + +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from" + " the primary server: %s", remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; + } + + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again.", + remote_slot->name)); + pfree(cmd.data); + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old lsns. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +static void +drop_initiated_sync_slots() +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server and belongs to one of the DBs passed in. + */ +List * +get_local_synced_slot_names(Oid *dbids, uint32 dbcount) +{ + List *localSyncedSlots = NIL; + + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + for (int j = 0; j < dbcount; j++) + { + /* + * Add it to output list if this belongs to one of the + * worker's dbs. + */ + if (s->data.database == dbids[j]) + { + localSyncedSlots = lappend(localSyncedSlots, s); + break; + } + } + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(Oid *dbids, List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* + * Get the list of local slots for dbids managed by this worker, so that + * those not on remote could be dropped. + */ + local_slot_list = get_local_synced_slot_names(dbids, + MySlotSyncWorker->dbcount); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Construct Slot Query + * + * It constructs the query using dbids array in order to get failover + * logical slots information from the primary server. + */ +static void +construct_slot_query(StringInfo s, Oid *dbids) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i' and database IN "); + + appendStringInfoChar(s, '('); + for (int i = 0; i < MySlotSyncWorker->dbcount; i++) + { + char *dbname; + + if (i != 0) + appendStringInfoChar(s, ','); + + dbname = get_database_name(dbids[i]); + appendStringInfo(s, "%s", + quote_literal_cstr(dbname)); + pfree(dbname); + } + appendStringInfoChar(s, ')'); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + bool found = false; + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s = NULL; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (int i = 0; i < max_replication_slots && !found; i++) + { + s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->in_use) + found = (strcmp(NameStr(s->data.name), remote_slot->name) == 0); + } + LWLockRelease(ReplicationSlotControlLock); + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot-sync worker) and ready for sync, + * acquire and sync it. + */ + if (found && s->data.sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would move" + " it backwards", remote_slot->name)); + + ReplicationSlotRelease(); + CommitTransactionCommand(); + return; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (found && s->data.sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, emit WARNING. */ + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server for the dbids + * managed by this worker and then updates the slots locally as per the info + * received. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + Oid *dbids; + int count = 0; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* + * No more writes to dbcount and dbids by launcher after this until we + * release this lock. + */ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* + * Check dbcount before starting to sync. There is a possibility that + * dbids managed by this worker are no longer valid due to change in + * logical failover slots on the primary server. In that case, launcher + * will make dbcount=0 and will send SIGINT to shutdown this worker. Thus + * check dbcount before we proceed further. + */ + if (!MySlotSyncWorker->dbcount) + { + LWLockRelease(SlotSyncWorkerLock); + + /* Return and handle the interrupts in main loop */ + return false; + } + + /* Get dbids from dsa */ + dbids = (Oid *) dsa_get_address(dsa, MySlotSyncWorker->dbids_dp); + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s, dbids); + + elog(DEBUG2, "slot-sync worker %d's query:%s \n", MySlotSyncWorker->slot, + s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info from the primary server: %s", + res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + { + remote_slot->invalidated = RS_INVAL_NONE; + count++; + } + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot-sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + MySlotSyncWorker->last_update_time = now; + else if (TimestampDifferenceExceeds(MySlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* Drop local slots that no longer need to be synced. */ + drop_obsolete_slots(dbids, remote_slot_list); + + LWLockRelease(SlotSyncWorkerLock); + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot-sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect() +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot-sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot-sync worker %d is shutting" + " down on receiving SIGINT", MySlotSyncWorker->slot)); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + int worker_slot = DatumGetInt32(main_arg); + dsa_handle handle; + dsa_area *dsa; + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* + * Attach to the dynamic shared memory segment for the slot-sync worker + * and find its table of contents. + */ + memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsa_handle)); + dsa = dsa_attach(handle); + if (!dsa) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not map dynamic shared memory " + "segment for slot-sync worker"))); + + /* Primary initialization is complete. Now attach to our slot. */ + slotsync_worker_attach(worker_slot); + + ereport(LOG, + errmsg("replication slot-sync worker %d started", worker_slot)); + + before_shmem_exit(slotsync_worker_detach, PointerGetDatum(dsa)); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + drop_initiated_sync_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(dsa, wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot-sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 7036096653..3caebd51f4 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index b706046811..c910c9be96 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -77,12 +77,14 @@ Node *replication_parse_result; %token K_EXPORT_SNAPSHOT %token K_NOEXPORT_SNAPSHOT %token K_USE_SNAPSHOT +%token K_LIST_DBID_FOR_FAILOVER_SLOTS %type <node> command %type <node> base_backup start_replication start_logical_replication create_replication_slot drop_replication_slot alter_replication_slot identify_system read_replication_slot timeline_history show + list_dbid_for_failover_slots %type <list> generic_option_list %type <defelt> generic_option %type <uintval> opt_timeline @@ -117,6 +119,7 @@ command: | read_replication_slot | timeline_history | show + | list_dbid_for_failover_slots ; /* @@ -129,6 +132,16 @@ identify_system: } ; +/* + * LIST_DBID_FOR_FAILOVER_SLOTS + */ +list_dbid_for_failover_slots: + K_LIST_DBID_FOR_FAILOVER_SLOTS + { + $$ = (Node *) makeNode(ListDBForFailoverSlotsCmd); + } + ; + /* * READ_REPLICATION_SLOT %s */ diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 0b5ae23195..57ddc08dfc 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -129,6 +129,7 @@ ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } +LIST_DBID_FOR_FAILOVER_SLOTS { return K_LIST_DBID_FOR_FAILOVER_SLOTS; } LOGICAL { return K_LOGICAL; } SLOT { return K_SLOT; } TEMPORARY { return K_TEMPORARY; } @@ -306,6 +307,7 @@ replication_scanner_is_replication_command(void) case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: + case K_LIST_DBID_FOR_FAILOVER_SLOTS: /* Yes; push back the first token so we can parse later. */ repl_pushed_back_token = first_token; return true; diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 6ef66d649a..09097e5801 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -318,6 +318,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -647,12 +648,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 3565ca196f..4f37ca2bd8 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,40 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * It return RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + int slotno; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (slotno = 0; slotno < max_replication_slots; slotno++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[slotno]; + + if (strcmp(NameStr(s->data.name), NameStr(*name)) == 0) + { + PG_RETURN_INT16(s->data.invalidated); + } + } + LWLockRelease(ReplicationSlotControlLock); + + PG_RETURN_NULL(); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +267,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +449,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index d494ddfaad..a5f5fccde9 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -473,6 +473,73 @@ IdentifySystem(void) end_tup_output(tstate); } +/* + * Handle the LIST_DBID_FOR_FAILOVER_SLOTS command. + * + * Return the list of database-ids for failover logical slots. + * The returned list has no duplicates. + */ +static void +ListFailoverSlotsDbids(void) +{ + DestReceiver *dest; + TupOutputState *tstate; + TupleDesc tupdesc; + List *database_oids_list = NIL; + + dest = CreateDestReceiver(DestRemoteSimple); + + /* Need a tuple descriptor representing a single column */ + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "database_oid", + INT8OID, -1, 0); + + /* Prepare for projection of tuples */ + tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual); + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (int slotno = 0; slotno < max_replication_slots; slotno++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[slotno]; + Oid dboid; + bool failover_slot = false; + Datum values[1]; + bool nulls[1]; + + if (!slot->in_use) + continue; + + SpinLockAcquire(&slot->mutex); + + dboid = slot->data.database; + failover_slot = slot->data.failover; + + SpinLockRelease(&slot->mutex); + + if (!failover_slot || SlotIsPhysical(slot)) + continue; + + /* Skip this slot if the database OID is already in the list. */ + if (list_member_oid(database_oids_list, dboid)) + continue; + + /* Add the database OID to the list */ + database_oids_list = lappend_oid(database_oids_list, dboid); + + values[0] = Int64GetDatum(dboid); + nulls[0] = (dboid == InvalidOid); + + /* Send it to dest */ + do_tup_output(tstate, values, nulls); + } + LWLockRelease(ReplicationSlotControlLock); + + /* Clean up the list */ + list_free(database_oids_list); + + end_tup_output(tstate); +} + /* Handle READ_REPLICATION_SLOT command */ static void ReadReplicationSlot(ReadReplicationSlotCmd *cmd) @@ -1255,7 +1322,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* @@ -2104,6 +2171,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_ListDBForFailoverSlotsCmd: + cmdtag = "LIST_DBID_FOR_FAILOVER_SLOTS"; + set_ps_display(cmdtag); + ListFailoverSlotsDbids(); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f6e2ec82c1..4affc1c861 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot-sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot-sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index fb23e8b7de..6f9b275835 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + true, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL @@ -3518,6 +3530,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_slotsync_workers", + PGC_POSTMASTER, + REPLICATION_STANDBY, + gettext_noop("Maximum number of slot synchronization workers " + "on a standby."), + NULL, + }, + &max_slotsync_workers, + 2, 0, 50, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..d83647889d 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,8 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary +#max_slotsync_workers = 2 # maximum number of slot synchronization workers # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ca626dff13..d20eff570b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11078,14 +11078,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..236b89d8d5 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,12 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT int max_slotsync_workers; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +34,7 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..261c560190 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,8 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbinfo_for_failover_slots_fn walrcv_get_dbinfo_for_failover_slots; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +455,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbinfo_for_failover_slots(conn) \ + WalReceiverFunctions->walrcv_get_dbinfo_for_failover_slots(conn) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..f899bffef4 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,11 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - /* Time at which this worker was launched. */ TimestampTz launch_time; @@ -53,6 +52,16 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,36 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot-sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot-sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorker +{ + LogicalWorkerHeader hdr; + + /* The slot in worker pool to which slot-sync worker is attached */ + int slot; + + /* Count of dbids slot-sync worker manages */ + uint32 dbcount; + + /* DSA for dbids */ + dsa_area *dbids_dsa; + + /* dsa_pointer for dbids slot-sync worker manages */ + dsa_pointer dbids_dp; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorker; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +273,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorker *MySlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(int slot); +extern void slotsync_worker_detach(int code, Datum arg); +extern List *get_local_synced_slot_names(Oid *dbids, uint32 dbcount); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +372,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +386,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..95fd9c180a 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..70bd65e2cf 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | on enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 0f184fb103..12d36122ad 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1430,6 +1430,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1509,6 +1510,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2310,6 +2312,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2566,6 +2569,7 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker SlruCtl SlruCtlData SlruErrorCause @@ -3013,6 +3017,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.34.1 [application/octet-stream] v32-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (125.7K, ../../CAJpy0uC2gYybBcvYCW68wvR-3akWnFbr7x66nZg7XTyjELW9iw@mail.gmail.com/4-v32-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From c8caee67aee89cfc0845b6943c4c1b23714cd5e6 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v32 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 3 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 61 +++- src/backend/replication/logical/worker.c | 40 ++- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 28 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 340 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/psql/describe.c | 9 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 2 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 43 files changed, 1223 insertions(+), 168 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index bd70ff2e4b..7136a925a9 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index a6fcac0824..d27c0543dc 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27485,7 +27485,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27500,8 +27500,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 35d738d576..24ad69c333 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..9c595ca3c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..7036096653 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,32 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { + char buf[100]; + + buf[0] = '\0'; + + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + strcat(buf, "twophase"); + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + { + if (buf[0] != '\0') + strcat(buf, " and "); + strcat(buf, "failover"); + } + ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that %s can be enabled", + MySubscription->name, buf))); should_exit = true; } } @@ -1412,7 +1429,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1732,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1746,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1768,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index ba67eb156f..8f3f353fa3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3949,6 +3949,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4484,6 +4485,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4541,16 +4544,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4559,11 +4583,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 99823df3c7..6ef66d649a 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -649,6 +656,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2135,3 +2167,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..3565ca196f 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) if (startlsn < moveto) { - SpinLockAcquire(&MyReplicationSlot->mutex); - MyReplicationSlot->data.restart_lsn = moveto; - SpinLockRelease(&MyReplicationSlot->mutex); + PhysicalConfirmReceivedLocation(moveto); retlsn = moveto; - - /* - * Dirty the slot so as it is written out at the next checkpoint. Note - * that the LSN position advanced may still be lost in the event of a - * crash, but this makes the data consistent after a clean shutdown. - */ - ReplicationSlotMarkDirty(); } return retlsn; @@ -679,6 +678,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +734,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +774,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index a3128874b2..8f4275f374 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -387,7 +387,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..d494ddfaad 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -260,7 +261,6 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch); static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); - /* Initialize walsender process before entering the main command loop */ void InitWalSender(void) @@ -974,12 +974,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1030,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1055,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1065,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1087,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1258,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1487,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1579,235 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Does this Wal Sender need to wake up logical walsender. + * + * Check if the physical slot of this walsender is specified in + * standby_slot_names GUC. + */ +static bool +WalSndWakeupNeeded() +{ + ListCell *lc; + List *standby_slots; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + return true; + } + + return false; +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndSlots(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Here logical walsender associated with failover logical slot waits + * for physical standbys corresponding to physical slots specified in + * standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndSlots(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to entering the loop in case we already know we have + * enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly + * interesting if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1828,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndSlots(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1843,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1882,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1930,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2097,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2030,7 +2315,7 @@ ProcessStandbyMessage(void) /* * Remember that a walreceiver just confirmed receipt of lsn `lsn`. */ -static void +void PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { bool changed = false; @@ -2049,6 +2334,9 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + + if (WalSndWakeupNeeded()) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); } /* @@ -2562,7 +2850,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3600,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3642,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..f6e2ec82c1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical standby confirmation in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 7605eff9b9..fb23e8b7de 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "walsenders only after specified replication slots " + "confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index bac94a338c..8b407b2f49 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6595,7 +6595,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6654,10 +6655,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n" ", subpasswordrequired AS \"%s\"\n" - ", subrunasowner AS \"%s\"\n", + ", subrunasowner AS \"%s\"\n" + ", subfailoverstate AS \"%s\"\n", gettext_noop("Origin"), gettext_noop("Password required"), - gettext_noop("Run as owner?")); + gettext_noop("Run as owner?"), + gettext_noop("Enable failover?")); appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 93742fc6ac..5f065e5c55 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3302,7 +3302,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 091f7e343c..ca626dff13 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11083,17 +11083,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..ecbd3526c5 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..7655437510 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,8 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 2a191830a8..6eefe59f2c 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -160,5 +160,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..0253752d03 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 87c1aee379..0f184fb103 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3856,6 +3857,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.34.1 ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-09 13:59 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-09 13:59 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/9/23 3:41 AM, Amit Kapila wrote: > On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand > <[email protected]> wrote: >> >>> Unrelated to above, if there is a user slot on standby with the same >>> name which the slot-sync worker is trying to create, then shall it >>> emit a warning and skip the sync of that slot or shall it throw an >>> error? >>> >> >> I'd vote for emit a warning and move on to the next slot if any. >> > > But then it could take time for users to know the actual problem and > they probably notice it after failover. Right, that's not appealing.... OTOH the slot has already been created manually on the standby so there is probably already a "use case" for it (that is probably unrelated to the failover story then). In V32, the following states have been introduced: " 'n': none for user slots, 'i': sync initiated for the slot but waiting for primary to catch up. 'r': ready for periodic syncs. " Should we introduce a new state that indicates that a sync slot creation has failed because the slot already existed? That would probably be simple to monitor instead of looking at the log file. > OTOH, if we throw an error > then probably they will come to know earlier because the slot sync > mechanism would be stopped. Right. > Do you have reasons to prefer giving a > WARNING and skipping creating such slots? My idea was that with a WARNING it won't block others slot creation (if any). > I expect this WARNING to > keep getting repeated in LOGs because the consecutive sync tries will > again generate a WARNING. > Yes. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-09 15:45 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 2 siblings, 2 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-09 15:45 UTC (permalink / raw) To: shveta malik <[email protected]>; Amit Kapila <[email protected]>; +Cc: Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/9/23 11:54 AM, shveta malik wrote: > > PFA v32 patches which has below changes: Thanks! > 7) Added warning for cases where a user-slot with the same name is > already present which slot-sync worker is trying to create. Sync for > such slots is skipped. I'm seeing assertion and segfault in this case due to ReplicationSlotRelease() in synchronize_one_slot(). Adding this extra check prior to it: - ReplicationSlotRelease(); + if (!(found && s->data.sync_state == SYNCSLOT_STATE_NONE)) + ReplicationSlotRelease(); make them disappear. > > Open Question: > 1) Currently I have put drop slot logic for slots with 'sync_state=i' > in slot-sync worker. Do we need to put it somewhere in promotion-logic > as well? Yeah I think so, because there is a time window when one could "use" the slot after the promotion and before it is removed. Producing things like: " 2023-11-09 15:16:50.294 UTC [2580462] LOG: dropped replication slot "logical_slot2" of dbid 5 as it was not sync-ready 2023-11-09 15:16:50.295 UTC [2580462] LOG: dropped replication slot "logical_slot3" of dbid 5 as it was not sync-ready 2023-11-09 15:16:50.297 UTC [2580462] LOG: dropped replication slot "logical_slot4" of dbid 5 as it was not sync-ready 2023-11-09 15:16:50.297 UTC [2580462] ERROR: replication slot "logical_slot5" is active for PID 2594628 " After the promotion one was able to use logical_slot5 and now we can now drop it. > Perhaps in WaitForWALToBecomeAvailable() where we call > XLogShutdownWalRcv after checking 'CheckForStandbyTrigger'. Thoughts? > You mean here? /* * Check to see if promotion is requested. Note that we do * this only after failure, so when you promote, we still * finish replaying as much as we can from archive and * pg_wal before failover. */ if (StandbyMode && CheckForStandbyTrigger()) { XLogShutdownWalRcv(); return XLREAD_FAIL; } If so, that sounds like a good place to me. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 03:31 shveta malik <[email protected]> parent: Drouvot, Bertrand <[email protected]> 1 sibling, 1 reply; 55+ messages in thread From: shveta malik @ 2023-11-10 03:31 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Thu, Nov 9, 2023 at 9:15 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 11/9/23 11:54 AM, shveta malik wrote: > > > > PFA v32 patches which has below changes: > > Thanks! > > > 7) Added warning for cases where a user-slot with the same name is > > already present which slot-sync worker is trying to create. Sync for > > such slots is skipped. > > I'm seeing assertion and segfault in this case due to ReplicationSlotRelease() > in synchronize_one_slot(). > > Adding this extra check prior to it: > > - ReplicationSlotRelease(); > + if (!(found && s->data.sync_state == SYNCSLOT_STATE_NONE)) > + ReplicationSlotRelease(); > > make them disappear. > Oh, I see. Thanks for pointing it out. I will fix it in the next version. > > > > Open Question: > > 1) Currently I have put drop slot logic for slots with 'sync_state=i' > > in slot-sync worker. Do we need to put it somewhere in promotion-logic > > as well? > > Yeah I think so, because there is a time window when one could "use" the slot > after the promotion and before it is removed. Producing things like: > > " > 2023-11-09 15:16:50.294 UTC [2580462] LOG: dropped replication slot "logical_slot2" of dbid 5 as it was not sync-ready > 2023-11-09 15:16:50.295 UTC [2580462] LOG: dropped replication slot "logical_slot3" of dbid 5 as it was not sync-ready > 2023-11-09 15:16:50.297 UTC [2580462] LOG: dropped replication slot "logical_slot4" of dbid 5 as it was not sync-ready > 2023-11-09 15:16:50.297 UTC [2580462] ERROR: replication slot "logical_slot5" is active for PID 2594628 > " > > After the promotion one was able to use logical_slot5 and now we can now drop it. Yes, I was suspicious about this small window which may allow others to use this slot, that is why I was thinking of putting it in the promotion flow and thus asked that question earlier. But the slot-sync worker may end up creating it again in case it has not exited. So we need to carefully decide at what all places we need to put 'not-in recovery' checks in slot-sync workers. In the previous version, synchronize_one_slot() had that check and it was skipping sync if '!RecoveryInProgress'. But I have removed that check in v32 thinking that the slots which the worker has already fetched from the primary, let them all get synced and exit after that nstead of syncing half and leaving rest. But now on rethinking, was the previous behaviour correct i.e. skip sync at that point onward where we see it is no longer in standby-mode while few of the slots have already been synced in that sync-cycle. Thoughts? > > > Perhaps in WaitForWALToBecomeAvailable() where we call > > XLogShutdownWalRcv after checking 'CheckForStandbyTrigger'. Thoughts? > > > > You mean here? > > /* > * Check to see if promotion is requested. Note that we do > * this only after failure, so when you promote, we still > * finish replaying as much as we can from archive and > * pg_wal before failover. > */ > if (StandbyMode && CheckForStandbyTrigger()) > { > XLogShutdownWalRcv(); > return XLREAD_FAIL; > } > yes, here. > If so, that sounds like a good place to me. okay. Will add it. > > Regards, > > -- > Bertrand Drouvot > PostgreSQL Contributors Team > RDS Open Source Databases > Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 05:41 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Amit Kapila @ 2023-11-10 05:41 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thu, Nov 9, 2023 at 7:29 PM Drouvot, Bertrand <[email protected]> wrote: > > On 11/9/23 3:41 AM, Amit Kapila wrote: > > On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand > > <[email protected]> wrote: > >> > >>> Unrelated to above, if there is a user slot on standby with the same > >>> name which the slot-sync worker is trying to create, then shall it > >>> emit a warning and skip the sync of that slot or shall it throw an > >>> error? > >>> > >> > >> I'd vote for emit a warning and move on to the next slot if any. > >> > > > > But then it could take time for users to know the actual problem and > > they probably notice it after failover. > > Right, that's not appealing.... > > OTOH the slot has already been created manually on the standby so there is > probably already a "use case" for it (that is probably unrelated to the > failover story then). > > In V32, the following states have been introduced: > > " > 'n': none for user slots, > 'i': sync initiated for the slot but waiting for primary to catch up. > 'r': ready for periodic syncs. > " > > Should we introduce a new state that indicates that a sync slot creation > has failed because the slot already existed? That would probably > be simple to monitor instead of looking at the log file. > Are you saying that we change the state of the already existing slot on standby? And, such a state would indicate that we are trying to sync the slot with the same name from the primary. Is that what you have in mind? If so, it appears quite odd to me to have such a state and also set it in some unrelated slot that just has the same name. I understand your point that we can allow other slots to proceed but it is also important to not create any sort of inconsistency that can surprise user after failover. Also, the current coding doesn't ensure we will always give WARNING. If we see the below code that deals with this WARNING, + /* User created slot with the same name exists, emit WARNING. */ + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); I think this is not a solid check to ensure that the slot existed before. Because it could be created as soon as the slot sync worker invokes ReplicationSlotCreate() here. So, depending on the timing, we can either get an ERROR or WARNING. I feel giving an ERROR in this case should be okay. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 07:20 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-10 07:20 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/10/23 6:41 AM, Amit Kapila wrote: > On Thu, Nov 9, 2023 at 7:29 PM Drouvot, Bertrand > <[email protected]> wrote: > > Are you saying that we change the state of the already existing slot > on standby? Yes. > And, such a state would indicate that we are trying to > sync the slot with the same name from the primary. Is that what you > have in mind? Yes. > If so, it appears quite odd to me to have such a state > and also set it in some unrelated slot that just has the same name. > > I understand your point that we can allow other slots to proceed but > it is also important to not create any sort of inconsistency that can > surprise user after failover. But even if we ERROR out instead of emitting a WARNING, the user would still need to be notified/monitor such errors. I agree that then probably they will come to know earlier because the slot sync mechanism would be stopped but still it is not "guaranteed" (specially if there is no others "working" synced slots around.) And if they do not, then there is still a risk to use this slot after a failover thinking this is a "synced" slot. Giving more thoughts, what about using a dedicated/reserved naming convention for synced slot like synced_<primary_slot_name> or such and then: - prevent user to create sync_<whatever> slots on standby - sync <slot> on primary to sync_<slot> on standby - during failover, rename sync_<slot> to <slot> and if <slot> exists then emit a WARNING and keep sync_<slot> in place. That way both slots are still in place (the manually created <slot> and the sync_<slot<) and one could decide what to do with them. I don't think we'd need to worry about the cases where sync_ slot could be already created before we "prevent" such slots creation. Indeed I think they would not survive pg_upgrade before 17 -> 18 upgrades. So it looks like we'd be good as long as we are able to prevent sync_ slots creation on 17. Thoughts? > Also, the current coding doesn't ensure > we will always give WARNING. If we see the below code that deals with > this WARNING, > > + /* User created slot with the same name exists, emit WARNING. */ > + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) > + { > + ereport(WARNING, > + errmsg("not synchronizing slot %s; it is a user created slot", > + remote_slot->name)); > + } > + /* Otherwise create the slot first. */ > + else > + { > + TransactionId xmin_horizon = InvalidTransactionId; > + ReplicationSlot *slot; > + > + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > + remote_slot->two_phase, false); > > I think this is not a solid check to ensure that the slot existed > before. Because it could be created as soon as the slot sync worker > invokes ReplicationSlotCreate() here. Agree. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 07:55 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Amit Kapila @ 2023-11-10 07:55 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Fri, Nov 10, 2023 at 12:50 PM Drouvot, Bertrand <[email protected]> wrote: > > On 11/10/23 6:41 AM, Amit Kapila wrote: > > On Thu, Nov 9, 2023 at 7:29 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > Are you saying that we change the state of the already existing slot > > on standby? > > Yes. > > > And, such a state would indicate that we are trying to > > sync the slot with the same name from the primary. Is that what you > > have in mind? > > Yes. > > > If so, it appears quite odd to me to have such a state > > and also set it in some unrelated slot that just has the same name. > > > > > I understand your point that we can allow other slots to proceed but > > it is also important to not create any sort of inconsistency that can > > surprise user after failover. > > But even if we ERROR out instead of emitting a WARNING, the user would still > need to be notified/monitor such errors. I agree that then probably they will > come to know earlier because the slot sync mechanism would be stopped but still > it is not "guaranteed" (specially if there is no others "working" synced slots > around.) > > And if they do not, then there is still a risk to use this slot after a > failover thinking this is a "synced" slot. > I think this is another reason that probably giving ERROR has better chances for the user to notice before failover. IF knowing such errors user still proceeds with the failover, the onus is on her. We can probably document this hazard along with the failover feature so that users are aware that they either need to be careful while creating slots on standby or consult ERROR logs. I guess we can even make it visible in the view also. > Giving more thoughts, what about using a dedicated/reserved naming convention for > synced slot like synced_<primary_slot_name> or such and then: > > - prevent user to create sync_<whatever> slots on standby > - sync <slot> on primary to sync_<slot> on standby > - during failover, rename sync_<slot> to <slot> and if <slot> exists then > emit a WARNING and keep sync_<slot> in place. > > That way both slots are still in place (the manually created <slot> and > the sync_<slot<) and one could decide what to do with them. > Hmm, I think after failover, users need to rename all slots or we need to provide a way to rename them so that they can be used by subscribers which sounds like much more work. > > Also, the current coding doesn't ensure > > we will always give WARNING. If we see the below code that deals with > > this WARNING, > > > > + /* User created slot with the same name exists, emit WARNING. */ > > + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) > > + { > > + ereport(WARNING, > > + errmsg("not synchronizing slot %s; it is a user created slot", > > + remote_slot->name)); > > + } > > + /* Otherwise create the slot first. */ > > + else > > + { > > + TransactionId xmin_horizon = InvalidTransactionId; > > + ReplicationSlot *slot; > > + > > + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > > + remote_slot->two_phase, false); > > > > I think this is not a solid check to ensure that the slot existed > > before. Because it could be created as soon as the slot sync worker > > invokes ReplicationSlotCreate() here. > > Agree. > So, having a concrete check to give WARNING would require some more logic which I don't think is a good idea to handle this boundary case. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 07:59 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-10 07:59 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thu, Nov 9, 2023 at 9:15 PM Drouvot, Bertrand <[email protected]> wrote: > > > You mean here? > > /* > * Check to see if promotion is requested. Note that we do > * this only after failure, so when you promote, we still > * finish replaying as much as we can from archive and > * pg_wal before failover. > */ > if (StandbyMode && CheckForStandbyTrigger()) > { > XLogShutdownWalRcv(); > return XLREAD_FAIL; > } > > If so, that sounds like a good place to me. > One more thing to think about is whether we want to shut down syncslot workers as well on promotion similar to walreceiver? Because we don't want them to even attempt once to sync after promotion. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 08:15 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-10 08:15 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/10/23 8:55 AM, Amit Kapila wrote: > On Fri, Nov 10, 2023 at 12:50 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> But even if we ERROR out instead of emitting a WARNING, the user would still >> need to be notified/monitor such errors. I agree that then probably they will >> come to know earlier because the slot sync mechanism would be stopped but still >> it is not "guaranteed" (specially if there is no others "working" synced slots >> around.) > >> >> And if they do not, then there is still a risk to use this slot after a >> failover thinking this is a "synced" slot. >> > > I think this is another reason that probably giving ERROR has better > chances for the user to notice before failover. IF knowing such errors > user still proceeds with the failover, the onus is on her. Agree. My concern is more when they don't know about the error. > We can > probably document this hazard along with the failover feature so that > users are aware that they either need to be careful while creating > slots on standby or consult ERROR logs. I guess we can even make it > visible in the view also. Yeah. >> Giving more thoughts, what about using a dedicated/reserved naming convention for >> synced slot like synced_<primary_slot_name> or such and then: >> >> - prevent user to create sync_<whatever> slots on standby >> - sync <slot> on primary to sync_<slot> on standby >> - during failover, rename sync_<slot> to <slot> and if <slot> exists then >> emit a WARNING and keep sync_<slot> in place. >> >> That way both slots are still in place (the manually created <slot> and >> the sync_<slot<) and one could decide what to do with them. >> > > Hmm, I think after failover, users need to rename all slots or we need > to provide a way to rename them so that they can be used by > subscribers which sounds like much more work. Agree that's much more work for the subscriber case. Maybe that's not worth the extra work. >>> Also, the current coding doesn't ensure >>> we will always give WARNING. If we see the below code that deals with >>> this WARNING, >>> >>> + /* User created slot with the same name exists, emit WARNING. */ >>> + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) >>> + { >>> + ereport(WARNING, >>> + errmsg("not synchronizing slot %s; it is a user created slot", >>> + remote_slot->name)); >>> + } >>> + /* Otherwise create the slot first. */ >>> + else >>> + { >>> + TransactionId xmin_horizon = InvalidTransactionId; >>> + ReplicationSlot *slot; >>> + >>> + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, >>> + remote_slot->two_phase, false); >>> >>> I think this is not a solid check to ensure that the slot existed >>> before. Because it could be created as soon as the slot sync worker >>> invokes ReplicationSlotCreate() here. >> >> Agree. >> > > So, having a concrete check to give WARNING would require some more > logic which I don't think is a good idea to handle this boundary case. > Yeah good point, agree to just error out in all the case then (if we discard the sync_ reserved wording proposal, which seems to be the case as probably not worth the extra work). Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-10 11:00 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-10 11:00 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/10/23 4:31 AM, shveta malik wrote: > On Thu, Nov 9, 2023 at 9:15 PM Drouvot, Bertrand > <[email protected]> wrote: >> Yeah I think so, because there is a time window when one could "use" the slot >> after the promotion and before it is removed. Producing things like: >> >> " >> 2023-11-09 15:16:50.294 UTC [2580462] LOG: dropped replication slot "logical_slot2" of dbid 5 as it was not sync-ready >> 2023-11-09 15:16:50.295 UTC [2580462] LOG: dropped replication slot "logical_slot3" of dbid 5 as it was not sync-ready >> 2023-11-09 15:16:50.297 UTC [2580462] LOG: dropped replication slot "logical_slot4" of dbid 5 as it was not sync-ready >> 2023-11-09 15:16:50.297 UTC [2580462] ERROR: replication slot "logical_slot5" is active for PID 2594628 >> " >> >> After the promotion one was able to use logical_slot5 and now we can now drop it. > > Yes, I was suspicious about this small window which may allow others > to use this slot, that is why I was thinking of putting it in the > promotion flow and thus asked that question earlier. But the slot-sync > worker may end up creating it again in case it has not exited. Sorry, there is a typo up-thread, I meant "After the promotion one was able to use logical_slot5 and now we can NOT drop it.". We can not drop it because it is in use. > So we > need to carefully decide at what all places we need to put 'not-in > recovery' checks in slot-sync workers. In the previous version, > synchronize_one_slot() had that check and it was skipping sync if > '!RecoveryInProgress'. But I have removed that check in v32 thinking > that the slots which the worker has already fetched from the primary, > let them all get synced and exit after that nstead of syncing half > and leaving rest. But now on rethinking, was the previous behaviour > correct i.e. skip sync at that point onward where we see it is no > longer in standby-mode while few of the slots have already been synced > in that sync-cycle. Thoughts? > I think we still need to think/discuss the promotion flow. I think we would need to have the slot sync worker shutdown during the promotion (as suggested by Amit in [1]) but before that let the sync slot worker knows it is now acting during promotion. Something like: - let the sync worker know it is now acting under promotion - do what needs to be done while acting under promotion - shutdown the sync worker That way we would avoid any "risk" of having the sync worker doing something we don't expect while not in recovery anymore. Regarding "do what needs to be done while acting under promotion": - Ensure all slots in 'r' state are synced - drop slots that are in 'i' state Thoughts? [1]: https://www.postgresql.org/message-id/CAA4eK1J2Pc%3D5TOgty5u4bp--y7ZHaQx3_2eWPL%3DVPJ7A_0JF2g%40mail... Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-13 00:49 Zhijie Hou (Fujitsu) <[email protected]> parent: shveta malik <[email protected]> 2 siblings, 1 reply; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-13 00:49 UTC (permalink / raw) To: shveta malik <[email protected]>; Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thursday, November 9, 2023 6:54 PM shveta malik <[email protected]> wrote: > > > PFA v32 patches which has below changes: Thanks for updating the patch. Here are few comments: 1. Do we need to update the slot upgrade code in pg_upgrade to upgrade the slot's failover property as well ? 2. The check for wal_level < WAL_LEVEL_LOGICAL. It seems the existing codes disallow slot creation if wal_level is not sufficient, I am thinking we might need similar check in slot sync worker. Otherwise, the synced slot could not be used after standby promotion. Best Regards, Hou zj ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 04:00 shveta malik <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-13 04:00 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Mon, Nov 13, 2023 at 6:19 AM Zhijie Hou (Fujitsu) <[email protected]> wrote: > > On Thursday, November 9, 2023 6:54 PM shveta malik <[email protected]> wrote: > > > > > > PFA v32 patches which has below changes: > > Thanks for updating the patch. > > Here are few comments: > > > 2. > The check for wal_level < WAL_LEVEL_LOGICAL. > > It seems the existing codes disallow slot creation if wal_level is not > sufficient, I am thinking we might need similar check in slot sync worker. > Otherwise, the synced slot could not be used after standby promotion. > Yes, I agree. We should add this check. > Best Regards, > Hou zj ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 04:24 shveta malik <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 1 reply; 55+ messages in thread From: shveta malik @ 2023-11-13 04:24 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Thu, Nov 9, 2023 at 8:56 AM Amit Kapila <[email protected]> wrote: > > On Thu, Nov 9, 2023 at 8:11 AM Amit Kapila <[email protected]> wrote: > > > > On Wed, Nov 8, 2023 at 8:09 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > > > > Unrelated to above, if there is a user slot on standby with the same > > > > name which the slot-sync worker is trying to create, then shall it > > > > emit a warning and skip the sync of that slot or shall it throw an > > > > error? > > > > > > > > > > I'd vote for emit a warning and move on to the next slot if any. > > > > > > > But then it could take time for users to know the actual problem and > > they probably notice it after failover. OTOH, if we throw an error > > then probably they will come to know earlier because the slot sync > > mechanism would be stopped. Do you have reasons to prefer giving a > > WARNING and skipping creating such slots? I expect this WARNING to > > keep getting repeated in LOGs because the consecutive sync tries will > > again generate a WARNING. > > > > Apart from the above, I would like to discuss the slot sync work > distribution strategy of this patch. The current implementation as > explained in the commit message [1] works well if the slots belong to > multiple databases. It is clear from the data in emails [2][3][4] that > having more workers really helps if the slots belong to multiple > databases. But I think if all the slots belong to one or very few > databases then such a strategy won't be as good. Now, on one hand, we > get very good numbers for a particular workload with the strategy used > in the patch but OTOH it may not be adaptable to various different > kinds of workloads. So, I have a question whether we should try to > optimize this strategy for various kinds of workloads or for the first > version let's use a single-slot sync-worker and then we can enhance > the functionality in later patches either in PG17 itself or in PG18 or > later versions. I can work on separating the patch. We can first focus on single worker design and then we can work on multi-worker design either immediately (if needed) or we can target it in the second draft of the patch. I would like to know the thoughts of others on this. One thing to note is that a lot of the complexity of > the patch is attributed to the multi-worker strategy which may still > not be efficient, so there is an argument to go with a simpler > single-slot sync-worker strategy and then enhance it in future > versions as we learn more about various workloads. It will also help > to develop this feature incrementally instead of doing all the things > in one go and taking a much longer time than it should. > Agreed. With multi-workers, a lot of complexity (dsa, locks etc) have come into play. We can decide better on our workload distribution strategy among workers once we have more clarity on different types of workloads. > > [1] - "The replication launcher on the physical standby queries > primary to get the list of dbids for failover logical slots. Once it > gets the dbids, if dbids < max_slotsync_workers, it starts only that > many workers, and if dbids > max_slotsync_workers, it starts > max_slotsync_workers and divides the work equally among them. Each > worker is then responsible to keep on syncing the logical slots > belonging to the DBs assigned to it. > > Each slot-sync worker will have its own dbids list. Since the upper > limit of this dbid-count is not known, it needs to be handled using > dsa. We initially allocated memory to hold 100 dbids for each worker. > If this limit is exhausted, we reallocate this memory with size > incremented again by 100." > > [2] - https://www.postgresql.org/message-id/CAJpy0uD2F43avuXy_yQv7Wa3kpUwioY_Xn955xdmd6vX0ME6%3Dg%40mail.g... > [3] - https://www.postgresql.org/message-id/CAFPTHDZw2G3Pax0smymMjfPqdPcZhMWo36f9F%2BTwNTs0HFxK%2Bw%40mail... > [4] - https://www.postgresql.org/message-id/CAJpy0uD%3DDevMxTwFVsk_%3DxHqYNH8heptwgW6AimQ9fbRmx4ioQ%40mail... > > -- > With Regards, > Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 05:32 Ajin Cherian <[email protected]> parent: shveta malik <[email protected]> 2 siblings, 1 reply; 55+ messages in thread From: Ajin Cherian @ 2023-11-13 05:32 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Alvaro Herrera <[email protected]> On Thu, Nov 9, 2023 at 9:54 PM shveta malik <[email protected]> wrote: > > PFA v32 patches which has below changes: Testing with this patch, I see that if the failover enabled slot is invalidated on the primary, then the corresponding synced slot is not invalidated on the standby. Instead, I see that it continuously gets the below error: " WARNING: not synchronizing slot sub; synchronization would move it backwards" In the code, I see that: if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) { ereport(WARNING, errmsg("not synchronizing slot %s; synchronization would move" " it backwards", remote_slot->name)); ReplicationSlotRelease(); CommitTransactionCommand(); return; } If the restart_lsn of the remote slot is behind, then the local_slot_update() function is never called to set the invalidation status on the local slot. And for invalidated slots, restart_lsn is always NULL. regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 06:38 shveta malik <[email protected]> parent: Ajin Cherian <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: shveta malik @ 2023-11-13 06:38 UTC (permalink / raw) To: Ajin Cherian <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Mon, Nov 13, 2023 at 11:02 AM Ajin Cherian <[email protected]> wrote: > > On Thu, Nov 9, 2023 at 9:54 PM shveta malik <[email protected]> wrote: > > > > PFA v32 patches which has below changes: > Testing with this patch, I see that if the failover enabled slot is > invalidated on the primary, then the corresponding synced slot is not > invalidated on the standby. Instead, I see that it continuously gets > the below error: > " WARNING: not synchronizing slot sub; synchronization would move it backwards" > > In the code, I see that: > if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) > { > ereport(WARNING, > errmsg("not synchronizing slot %s; synchronization > would move" > " it backwards", remote_slot->name)); > > ReplicationSlotRelease(); > CommitTransactionCommand(); > return; > } > > If the restart_lsn of the remote slot is behind, then the > local_slot_update() function is never called to set the invalidation > status on the local slot. And for invalidated slots, restart_lsn is > always NULL. > Okay. Thanks for testing Ajin. I think it needs a fix wherein we set the local-slot's invalidation status (provided it is not invalidated already) from the remote slot before this check itself. And if the slot is invalidated locally (either by itself) or by primary_slot being invalidated, then we should skip the sync. I will fix this in the next version. thanks Shveta ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 09:12 Ajin Cherian <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: Ajin Cherian @ 2023-11-13 09:12 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Alvaro Herrera <[email protected]> On Mon, Nov 13, 2023 at 5:38 PM shveta malik <[email protected]> wrote: > Okay. Thanks for testing Ajin. I think it needs a fix wherein we set > the local-slot's invalidation status (provided it is not invalidated > already) from the remote slot before this check itself. And if the > slot is invalidated locally (either by itself) or by primary_slot > being invalidated, then we should skip the sync. I will fix this in > the next version. Yes, that works. Another bug I see in my testing is that pg_get_slot_invalidation_cause() does not release the LOCK if it finds the slot it is searching for. regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-13 09:53 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-13 09:53 UTC (permalink / raw) To: shveta malik <[email protected]>; Amit Kapila <[email protected]>; +Cc: Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/13/23 5:24 AM, shveta malik wrote: > On Thu, Nov 9, 2023 at 8:56 AM Amit Kapila <[email protected]> wrote: >> >> Apart from the above, I would like to discuss the slot sync work >> distribution strategy of this patch. The current implementation as >> explained in the commit message [1] works well if the slots belong to >> multiple databases. It is clear from the data in emails [2][3][4] that >> having more workers really helps if the slots belong to multiple >> databases. But I think if all the slots belong to one or very few >> databases then such a strategy won't be as good. Now, on one hand, we >> get very good numbers for a particular workload with the strategy used >> in the patch but OTOH it may not be adaptable to various different >> kinds of workloads. So, I have a question whether we should try to >> optimize this strategy for various kinds of workloads or for the first >> version let's use a single-slot sync-worker and then we can enhance >> the functionality in later patches either in PG17 itself or in PG18 or >> later versions. > > I can work on separating the patch. We can first focus on single > worker design and then we can work on multi-worker design either > immediately (if needed) or we can target it in the second draft of the > patch. I would like to know the thoughts of others on this. If we need to put more thoughts on the workers distribution strategy then I also think it's better to focus on a single worker and then improve/discuss a distribution design later on. > > One thing to note is that a lot of the complexity of >> the patch is attributed to the multi-worker strategy which may still >> not be efficient, so there is an argument to go with a simpler >> single-slot sync-worker strategy and then enhance it in future >> versions as we learn more about various workloads. It will also help >> to develop this feature incrementally instead of doing all the things >> in one go and taking a much longer time than it should. > > Agreed. With multi-workers, a lot of complexity (dsa, locks etc) have > come into play. We can decide better on our workload distribution > strategy among workers once we have more clarity on different types of > workloads. > Agreed. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-13 13:57 Zhijie Hou (Fujitsu) <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 2 replies; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-13 13:57 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand <[email protected]> wrote: > > > >>> Also, the current coding doesn't ensure we will always give WARNING. > >>> If we see the below code that deals with this WARNING, > >>> > >>> + /* User created slot with the same name exists, emit WARNING. */ > >>> + else if (found && s->data.sync_state == SYNCSLOT_STATE_NONE) > >>> + { > >>> + ereport(WARNING, > >>> + errmsg("not synchronizing slot %s; it is a user created slot", > >>> + remote_slot->name)); > >>> + } > >>> + /* Otherwise create the slot first. */ > >>> + else > >>> + { > >>> + TransactionId xmin_horizon = InvalidTransactionId; > >>> + ReplicationSlot *slot; > >>> + > >>> + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > >>> + remote_slot->two_phase, false); > >>> > >>> I think this is not a solid check to ensure that the slot existed > >>> before. Because it could be created as soon as the slot sync worker > >>> invokes ReplicationSlotCreate() here. > >> > >> Agree. > >> > > > > So, having a concrete check to give WARNING would require some more > > logic which I don't think is a good idea to handle this boundary case. > > > > Yeah good point, agree to just error out in all the case then (if we discard the > sync_ reserved wording proposal, which seems to be the case as probably not > worth the extra work). Thanks for the discussion! Here is the V33 patch set which includes the following changes: 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. 2) Raise error if user slot with same name already exists on standby. 3) Hold spinlock when updating the porperty of the replication slot or when reading the slot's info without acuqiring it. 4) Fixed the bug: - if slot is invalidated on standby but standby is restarted immediately after that, then it fails to recreate that slot and instead end up syncing it again. It is fixed by checking the invalidated flag after acquiring the slot and skip syncing for invalidated slots. 5) Fixed the bugs reported by Ajin[1][2]. 6) Removed some unused variables. Thanks Shveta for working on 1) 2) 4) and 5). Thanks Ajin for testing 5). --- TODO There are few pending comments and bugs have not been addressed, I will work on them in next version: 1) Comments posted by me[3]: 2) Shutdown the slotsync worker on promotion and probably let slotsync do necessary cleanups[4] 3) Consider documenting the hazard that create slot on standby may cause ERRORs in the log and consider making it visible in the view. 4) One bug found internally: If we give non-existing dbname in primary_conninfo on standby, it should be handled gracefully, launcher should skip slots-synchronization. 5) One bug found internally: when wait_for_primary_slot_catchup is doing WaitLatch and I stop the standby using: ./pg_ctl -D ../../standbydb/ -l standby.log stop it does not come out of wait and shutdown hangs. [1] https://www.postgresql.org/message-id/CAFPTHDZNV_HFAXULkaJOv_MMtLukCzDEgTaixxBwjEO_0Jg-kg%40mail.gma... [2] https://www.postgresql.org/message-id/CAFPTHDa5C_vHQbeqemToyucWySB0kEFbdS2WOA0PB%2BGSei2v7A%40mail.g... [3] https://www.postgresql.org/message-id/OS0PR01MB571652CCD42F1D08D5BD69D494B3A%40OS0PR01MB5716.jpnprd0... [4] https://www.postgresql.org/message-id/64056e35-1916-461c-a816-26e40ffde3a0%40gmail.com Best Regards, Hou zj Attachments: [application/octet-stream] v33-0003-Allow-slot-sync-workers-to-wait-for-the-cascadin.patch (7.9K, ../../OS0PR01MB5716CE0729CEB3B5994A954194B3A@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v33-0003-Allow-slot-sync-workers-to-wait-for-the-cascadin.patch) download | inline diff: From 587919a5b0ca1203f5d0a42b3bb69ac8fc189430 Mon Sep 17 00:00:00 2001 From: Hou Zhijie <[email protected]> Date: Mon, 13 Nov 2023 21:44:47 +0800 Subject: [PATCH v33 3/3] Allow slot-sync workers to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync workers. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 1cee99a761..d76710727b 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -82,6 +82,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -544,6 +547,52 @@ drop_obsolete_slots(Oid *dbids, List *remote_slot_list) } } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Construct Slot Query * @@ -767,6 +816,7 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; Oid *dbids; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -841,6 +891,9 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for lsns and xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -887,6 +940,17 @@ synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) */ drop_obsolete_slots(dbids, remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -945,12 +1009,18 @@ remote_connect() * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndSlots(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -968,7 +1038,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot-sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -984,7 +1055,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -1054,7 +1128,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index a5f5fccde9..cdf0afae13 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1678,7 +1678,7 @@ WalSndWakeupNeeded() * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndSlots(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1701,13 +1701,12 @@ WalSndRereadConfigAndSlots(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1774,10 +1773,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index ecbd3526c5..25c522993f 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndSlots(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.30.0.windows.2 [application/octet-stream] v33-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (125.7K, ../../OS0PR01MB5716CE0729CEB3B5994A954194B3A@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v33-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From 700abe99b2dee9f42a6af5b5a349e3d850265460 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v33 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 3 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 61 +++- src/backend/replication/logical/worker.c | 40 ++- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 28 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 340 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/psql/describe.c | 9 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 2 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 43 files changed, 1223 insertions(+), 168 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fc35a46e5e..f4fe1995d9 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d963f0a0a0..b44b222b58 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27515,7 +27515,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27530,8 +27530,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 8079f1cd7f..f6962dcc71 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..9c595ca3c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..7036096653 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,32 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { + char buf[100]; + + buf[0] = '\0'; + + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + strcat(buf, "twophase"); + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + { + if (buf[0] != '\0') + strcat(buf, " and "); + strcat(buf, "failover"); + } + ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that %s can be enabled", + MySubscription->name, buf))); should_exit = true; } } @@ -1412,7 +1429,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1732,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1746,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1768,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 52a9f136ab..0313bc2035 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 781aa43cc4..d9ec6cdf07 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -649,6 +656,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2129,3 +2161,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..3565ca196f 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) if (startlsn < moveto) { - SpinLockAcquire(&MyReplicationSlot->mutex); - MyReplicationSlot->data.restart_lsn = moveto; - SpinLockRelease(&MyReplicationSlot->mutex); + PhysicalConfirmReceivedLocation(moveto); retlsn = moveto; - - /* - * Dirty the slot so as it is written out at the next checkpoint. Note - * that the LSN position advanced may still be lost in the event of a - * crash, but this makes the data consistent after a clean shutdown. - */ - ReplicationSlotMarkDirty(); } return retlsn; @@ -679,6 +678,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +734,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +774,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..d494ddfaad 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -260,7 +261,6 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch); static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); - /* Initialize walsender process before entering the main command loop */ void InitWalSender(void) @@ -974,12 +974,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1030,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1055,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1065,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1087,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1258,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1487,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1579,235 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Does this Wal Sender need to wake up logical walsender. + * + * Check if the physical slot of this walsender is specified in + * standby_slot_names GUC. + */ +static bool +WalSndWakeupNeeded() +{ + ListCell *lc; + List *standby_slots; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + return true; + } + + return false; +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndSlots(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Here logical walsender associated with failover logical slot waits + * for physical standbys corresponding to physical slots specified in + * standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndSlots(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to entering the loop in case we already know we have + * enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly + * interesting if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1828,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndSlots(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1843,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1882,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1930,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2097,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2030,7 +2315,7 @@ ProcessStandbyMessage(void) /* * Remember that a walreceiver just confirmed receipt of lsn `lsn`. */ -static void +void PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { bool changed = false; @@ -2049,6 +2334,9 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + + if (WalSndWakeupNeeded()) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); } /* @@ -2562,7 +2850,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3600,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3642,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..f6e2ec82c1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical standby confirmation in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..b671615c39 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "walsenders only after specified replication slots " + "confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index bac94a338c..8b407b2f49 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6595,7 +6595,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6654,10 +6655,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n" ", subpasswordrequired AS \"%s\"\n" - ", subrunasowner AS \"%s\"\n", + ", subrunasowner AS \"%s\"\n" + ", subfailoverstate AS \"%s\"\n", gettext_noop("Origin"), gettext_noop("Password required"), - gettext_noop("Run as owner?")); + gettext_noop("Run as owner?"), + gettext_noop("Enable failover?")); appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 93742fc6ac..5f065e5c55 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3302,7 +3302,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index bd0b8873d3..51faf936cb 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11087,17 +11087,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..ecbd3526c5 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..7655437510 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,8 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..0253752d03 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bf50a32119..89f34aacbc 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3860,6 +3861,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.30.0.windows.2 [application/octet-stream] v33-0002-Add-logical-slot-sync-capability-to-the-physical.patch (121.4K, ../../OS0PR01MB5716CE0729CEB3B5994A954194B3A@OS0PR01MB5716.jpnprd01.prod.outlook.com/4-v33-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From e5e9f00b8030764d451f3dfc121d70acdeed1e78 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v33 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker(s) on the standby server ping the primary at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. GUC 'max_slotsync_workers' defines the maximum number of slot-sync workers on the standby. This parameter can only be set at server start. The replication launcher on the physical standby queries primary to get the list of dbids for failover logical slots. Once it gets the dbids, if dbids < max_slotsync_workers, it starts only that many workers and if dbids > max_slotsync_workers, it starts max_slotsync_workers and divides the work equally among them. Each worker is then responsible to keep on syncing the logical slots belonging to the DBs assigned to it. Each slot-sync worker will have its own dbids list. Since the upper limit of this dbid-count is not known, it needs to be handled using dsa. We initially allocate memory to hold 100 dbids for each worker. If this limit is exhausted, we reallocate this memory with size incremented again by 100. The nap time of worker is tuned according to the activity on the primary. Each worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. Each worker uses one slot (first one assigned to it) for monitoring purpose. If there is no change in lsn of that slot for some threshold time, nap time is increased to 10sec and as soon as a change is observed, nap time is reduced back to 10ms. The logical slots created by slot-sync workers on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). --- doc/src/sgml/config.sgml | 56 +- doc/src/sgml/system-views.sgml | 17 + src/backend/access/transam/xlogrecovery.c | 6 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 89 ++ src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 1033 ++++++++++++++-- src/backend/replication/logical/logical.c | 12 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1090 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/repl_gram.y | 13 + src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 17 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 76 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 25 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 8 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 30 + src/include/replication/worker_internal.h | 62 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 36 files changed, 2631 insertions(+), 148 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f4fe1995d9..1e4eaee32b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,14 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4888,50 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-max-slotsync-workers" xreflabel="max_slotsync_workers"> + <term><varname>max_slotsync_workers</varname> (<type>integer</type>) + <indexterm> + <primary><varname>max_slotsync_workers</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + Specifies maximum number of slot synchronization workers. + </para> + <para> + Slot synchronization workers are taken from the pool defined by + <varname>max_worker_processes</varname>. + </para> + <para> + The default value is 2. This parameter can only be set at server + start. + </para> + <para> + The slot-sync workers are needed for synchronization of logical replication + slots from the primary server to the physical standby so that logical + subscribers are not blocked after failover. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..cc1adf9127 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,23 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..ce69917c2f 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -50,6 +50,7 @@ #include "postmaster/startup.h" #include "replication/slot.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * this only after failure, so when you promote, we still * finish replaying as much as we can from archive and * pg_wal before failover. + * + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary + * server to catch up. */ if (StandbyMode && CheckForStandbyTrigger()) { XLogShutdownWalRcv(); + slotsync_drop_initiated_slots(); return XLREAD_FAIL; } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 9c595ca3c9..0bc3264d20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..d217d38641 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,8 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static List *libpqrcv_get_dbinfo_for_failover_slots(WalReceiverConn *conn); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +103,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbinfo_for_failover_slots = libpqrcv_get_dbinfo_for_failover_slots, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +418,90 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get DB info for logical failover slots + * + * It gets the DBIDs for failover logical slots from the primary server. + * The list returned by LIST_DBID_FOR_FAILOVER_SLOTS has no duplicates. + */ +static List * +libpqrcv_get_dbinfo_for_failover_slots(WalReceiverConn *conn) +{ + PGresult *res; + List *slotlist = NIL; + int ntuples; + WalRcvFailoverSlotsData *slot_data; + + res = libpqrcv_PQexec(conn->streamConn, "LIST_DBID_FOR_FAILOVER_SLOTS"); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + ereport(ERROR, + (errmsg("could not receive logical failover slots dbinfo from the primary server: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + if (PQnfields(res) != 1) + { + int nfields = PQnfields(res); + + PQclear(res); + ereport(ERROR, + (errmsg("invalid response from the primary server"), + errdetail("Could not get logical failover slots dbinfo: got %d fields, " + "expected 1", nfields))); + } + + ntuples = PQntuples(res); + for (int i = 0; i < ntuples; i++) + { + slot_data = palloc0(sizeof(WalRcvFailoverSlotsData)); + if (!PQgetisnull(res, i, 0)) + slot_data->dboid = atooid(PQgetvalue(res, i, 0)); + + slotlist = lappend(slotlist, slot_data); + } + + PQclear(res); + + return slotlist; +} + +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* If multiple dbnames are specified, then the last one will be returned */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..7247d9dcc8 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -22,6 +22,7 @@ #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "catalog/pg_authid.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -57,6 +58,22 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +int max_slotsync_workers = 2; +bool enable_syncslot = true; + +/* + * Initial allocation size for dbids array for each SlotSyncWorker in dynamic + * shared memory. + */ +#define DB_PER_WORKER_ALLOC_INIT 100 + +/* + * Once initially allocated size is exhausted for dbids array, it is extended by + * DB_PER_WORKER_ALLOC_EXTRA size. + */ +#define DB_PER_WORKER_ALLOC_EXTRA 100 + +SlotSyncWorker *MySlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +87,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorker *ss_workers; /* slot-sync workers */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +120,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorker *worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +197,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slotsync workers. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +207,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +222,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorker *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +290,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +318,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +380,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,8 +409,8 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && - TimestampDifferenceExceeds(w->launch_time, now, + if (w->hdr.in_use && !w->hdr.proc && + TimestampDifferenceExceeds(w->hdr.launch_time, now, wal_receiver_timeout)) { elog(WARNING, @@ -437,10 +466,10 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; - worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.launch_time = now; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +486,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +539,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +551,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical rep workers and slot-sync workers. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +583,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +597,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +624,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +637,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +656,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +704,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +733,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +755,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +764,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +773,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +808,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +833,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +970,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot-sync workers. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot-sync workers to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -953,6 +1001,7 @@ void ApplyLauncherShmemInit(void) { bool found; + Size ssw_size; LogicalRepCtx = (LogicalRepCtxStruct *) ShmemInitStruct("Logical Replication Launcher Data", @@ -977,6 +1026,14 @@ ApplyLauncherShmemInit(void) SpinLockInit(&worker->relmutex); } } + + /* Allocate shared-memory for slot-sync workers pool now */ + ssw_size = mul_size(max_slotsync_workers, sizeof(SlotSyncWorker)); + LogicalRepCtx->ss_workers = (SlotSyncWorker *) + ShmemInitStruct("Replication slot-sync workers", ssw_size, &found); + + if (!found) + memset(LogicalRepCtx->ss_workers, 0, ssw_size); } /* @@ -1115,13 +1172,809 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot-sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorker *worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->slot = -1; + + if (DsaPointerIsValid(worker->dbids_dp)) + { + dsa_free(worker->dbids_dsa, worker->dbids_dp); + worker->dbids_dp = InvalidDsaPointer; + } + + if (worker->dbids_dsa) + { + dsa_detach(worker->dbids_dsa); + worker->dbids_dsa = NULL; + } + + worker->dbcount = 0; + worker->last_update_time = 0; +} + +/* + * Attach Slot-sync worker to worker-slot assigned by launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach(int slot) { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + Assert(slot >= 0 && slot < max_slotsync_workers); + MySlotSyncWorker = &LogicalRepCtx->ss_workers[slot]; + MySlotSyncWorker->slot = slot; + + if (!MySlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker slot %d is " + "empty, cannot attach", slot))); + } + + if (MySlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker slot %d is " + "already used by another worker, cannot attach", slot))); + } + + MySlotSyncWorker->hdr.proc = MyProc; + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + dsa_detach((dsa_area *) DatumGetPointer(arg)); + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + MySlotSyncWorker->hdr.in_use = false; + MySlotSyncWorker->hdr.proc = NULL; + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Slot-Sync worker find. + * + * Searches the slot-sync worker pool for the worker who manages the + * specified dbid. Because a worker can manage multiple dbs, also walk + * the db array of each worker to find the match. + * + * Returns NULL if no matching worker is found. + */ +static SlotSyncWorker * +slotsync_worker_find(Oid dbid) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* Search for an attached worker for a given dbid */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *w = &LogicalRepCtx->ss_workers[widx]; + Oid *dbids; + + if (!w->hdr.in_use) + continue; + + dbids = (Oid *) dsa_get_address(w->dbids_dsa, w->dbids_dp); + for (int dbidx = 0; dbidx < w->dbcount; dbidx++) + { + if (dbids[dbidx] == dbid) + return w; + } + + } + + return NULL; +} + +/* + * Setup the slot-sync worker. + * + * DSA is used for the dbids array. Because the maximum number of dbs a + * worker can manage is not known, initially enough memory for + * DB_PER_WORKER_ALLOC_INIT dbs is allocated. If this size is exhausted, + * it can be extended using dsa free and allocate routines. + */ +static dsa_handle +slotsync_worker_setup(SlotSyncWorker *worker) +{ + dsa_area *dbids_dsa; + dsa_pointer dbids_dp; + dsa_handle dbids_dsa_handle; + MemoryContext oldcontext; + + /* Prepare the new worker. */ + worker->hdr.launch_time = GetCurrentTimestamp(); + worker->hdr.in_use = true; + + /* + * 'proc' and 'slot' will be assigned in ReplSlotSyncWorkerMain when we + * attach this worker to a particular worker-pool slot + */ + worker->hdr.proc = NULL; + worker->slot = -1; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* Ensure the memory allocated by DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + dbids_dsa = dsa_create(LWTRANCHE_SLOTSYNC_DSA); + dsa_pin(dbids_dsa); + dsa_pin_mapping(dbids_dsa); + + dbids_dp = dsa_allocate0(dbids_dsa, DB_PER_WORKER_ALLOC_INIT * sizeof(Oid)); + + /* Set-up worker */ + worker->dbcount = 0; + worker->dbids_dsa = dbids_dsa; + worker->dbids_dp = dbids_dp; + + /* Get the handle. This is the one which can be passed to worker processes */ + dbids_dsa_handle = dsa_get_handle(dbids_dsa); + + elog(DEBUG1, "allocated dsa for slot-sync worker for dbcount: %d", + DB_PER_WORKER_ALLOC_INIT); + + MemoryContextSwitchTo(oldcontext); + + return dbids_dsa_handle; +} + +/* + * Slot-sync worker launch or reuse + * + * Start new slot-sync background worker from the pool of available workers + * limited by max_slotsync_workers count. If the worker pool is exhausted, + * reuse the existing worker with minimum number of dbs. The idea is to + * always distribute the dbs equally among launched workers. + * If initially allocated dbids array is exhausted for the selected worker, + * reallocate the dbids array with increased size and copy the existing + * dbids to it and assign the new one as well. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch_or_reuse(Oid dbid) +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + uint16 generation; + SlotSyncWorker *worker = NULL; + int worker_slot = -1; + dsa_handle handle; + Oid *dbids; + bool attach; + uint32 mindbcnt = PG_UINT32_MAX; + + Assert(OidIsValid(dbid)); + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + /* + * Find unused worker slot. If all the workers are currently in use, find + * the one with minimum number of dbs and use that. + */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *w = &LogicalRepCtx->ss_workers[widx]; + + if (!w->hdr.in_use) + { + worker = w; + worker_slot = widx; + break; + } + + if (w->dbcount < mindbcnt) + { + mindbcnt = w->dbcount; + worker = w; + worker_slot = widx; + } + } + + /* + * If worker is being reused, and there is vacancy in dbids array, just + * update dbids array and dbcount and we are done. But if dbids array is + * exhausted, reallocate dbids using dsa and copy the old dbids and assign + * the new one as well. + */ + if (worker->hdr.in_use) + { + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + if (worker->dbcount < DB_PER_WORKER_ALLOC_INIT) + { + dbids[worker->dbcount++] = dbid; + } + else + { + MemoryContext oldcontext; + uint32 alloc_count; + uint32 old_dbcnt; + Oid *old_dbids; + + /* Be sure any memory allocated by DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Remember the old dbids before we reallocate dsa. */ + old_dbcnt = worker->dbcount; + old_dbids = (Oid *) palloc0(worker->dbcount * sizeof(Oid)); + memcpy(old_dbids, dbids, worker->dbcount * sizeof(Oid)); + + alloc_count = old_dbcnt + DB_PER_WORKER_ALLOC_EXTRA; + + /* Free the existing dbids and allocate new with increased size */ + if (DsaPointerIsValid(worker->dbids_dp)) + dsa_free(worker->dbids_dsa, worker->dbids_dp); + + worker->dbids_dp = dsa_allocate0(worker->dbids_dsa, + alloc_count * sizeof(Oid)); + + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + /* Copy the existing dbids */ + worker->dbcount = old_dbcnt; + memcpy(dbids, old_dbids, old_dbcnt * sizeof(Oid)); + pfree(old_dbids); + + /* Assign new dbid */ + dbids[worker->dbcount++] = dbid; + + MemoryContextSwitchTo(oldcontext); + } + + LWLockRelease(SlotSyncWorkerLock); + + ereport(LOG, + (errmsg("added database %d to replication slot-sync " + "worker %d; dbcount now: %d", + dbid, worker_slot, worker->dbcount))); + return true; + } + + /* + * Initialise the worker and setup DSA for dbids array to hold + * DB_PER_WORKER_ALLOC_INIT dbs + */ + handle = slotsync_worker_setup(worker); + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + dbids[worker->dbcount++] = dbid; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + Assert(worker_slot >= 0); + snprintf(bgw.bgw_name, BGW_MAXLEN, + "replication slot-sync worker %d", worker_slot); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot-sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + bgw.bgw_main_arg = Int32GetDatum(worker_slot); + + memcpy(bgw.bgw_extra, &handle, sizeof(dsa_handle)); + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + /* + * If attach is done, log that the worker is managing dbid, else raise a + * warning + */ + if (attach) + ereport(LOG, + (errmsg("added database %d to replication slot-sync " + "worker %d; dbcount now: %d", + dbid, worker_slot, worker->dbcount))); + else + ereport(WARNING, + (errmsg("replication slot-sync worker failed to attach to " + "worker-pool slot %d", worker_slot))); + + return attach; +} + +/* + * Internal function to stop the slot-sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorker *worker) +{ + int slot = worker->slot; + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot-sync worker %d", + slot))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Stop all the slot-sync workers in use. + */ +static void +slotsync_workers_stop() +{ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + + if (worker && worker->hdr.in_use) + slotsync_worker_stop_internal(worker); + } +} + +/* + * Drop the local 'synced' slots for all the obsoleted dbs passed in. + * + * Helper function for slotsync_remove_obsolete_dbs(). + */ +static void +slotsync_drop_obsoleted_dbs_slots(List *dbids_list) +{ + List *local_slot_list = NIL; + ListCell *lc; + Oid *dbids; + uint32 idx = 0; + uint32 dbcount = list_length(dbids_list); + + dbids = palloc0(dbcount * sizeof(Oid)); + + foreach(lc, dbids_list) + { + Oid dbid = lfirst_oid(lc); + + dbids[idx++] = dbid; + } + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* Get the list of local 'synced' slots for the given dbids. */ + local_slot_list = get_local_synced_slot_names(dbids, dbcount); + + LWLockRelease(SlotSyncWorkerLock); + + pfree(dbids); + + foreach(lc, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } +} + +/* + * Slot-sync workers remove obsolete DBs from db-list + * + * If the DBIds fetched from the primary server are lesser than the ones being + * managed by slot-sync workers, remove extra dbs from worker's db-list. This + * may happen if some logical failover slots are removed on the primary server + * or are disabled for failover. + * Also remove the local 'synced' slots belonging to such dbs. + */ +static void +slotsync_remove_obsolete_dbs(List *remote_dbs) +{ + List *removed_dbs_list = NIL; + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + /* Traverse slot-sync-workers to validate the DBs */ + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + Oid *dbids; + + if (!worker->hdr.in_use) + continue; + + dbids = (Oid *) dsa_get_address(worker->dbids_dsa, worker->dbids_dp); + + for (int dbidx = 0; dbidx < worker->dbcount;) + { + Oid wdbid = dbids[dbidx]; + bool found = false; + ListCell *lc; + + /* Check if current DB is still present in remote-db-list */ + foreach(lc, remote_dbs) + { + WalRcvFailoverSlotsData *failover_slot_data = lfirst(lc); + + if (failover_slot_data->dboid == wdbid) + { + found = true; + dbidx++; + break; + } + } + + /* If not found, then delete this db from worker's db-list */ + if (!found) + { + if (dbidx < (worker->dbcount - 1)) + { + /* Shift the DBs and get rid of wdbid */ + memmove(&dbids[dbidx], &dbids[dbidx + 1], + (worker->dbcount - dbidx - 1) * sizeof(Oid)); + } + + worker->dbcount--; + + ereport(LOG, + (errmsg("removed database %d from replication slot-sync " + "worker %d; dbcount now: %d", + wdbid, worker->slot, worker->dbcount))); + + removed_dbs_list = lappend_oid(removed_dbs_list, wdbid); + } + + } + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If dbcount for any worker has become 0, shut it down. + * + * XXX: if needed in future, workers can be restarted in such a case to + * distribute the load. + */ + + for (int widx = 0; widx < max_slotsync_workers; widx++) + { + SlotSyncWorker *worker = &LogicalRepCtx->ss_workers[widx]; + + if (worker->hdr.in_use && !worker->dbcount) + slotsync_worker_stop_internal(worker); + } + + /* + * Drop local 'synced' slots for the dbs which are no longer eligible for + * slot-sync. + */ + if (removed_dbs_list) + { + slotsync_drop_obsoleted_dbs_slots(removed_dbs_list); + list_free(removed_dbs_list); + } +} + +/* + * Connect to the primary server for slotsync purpose and return the connection + * info. + */ +static WalReceiverConn * +slotsync_remote_connect(long *wait_time, bool *retry) +{ + WalReceiverConn *wrconn; + char *err; + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return NULL; + + if (max_slotsync_workers == 0) + return NULL; + + /* + * Since the above two GUCs are set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo) are + * compatible with slot synchronization. If not, issue warnings. + */ + + /* The primary_slot_name is not set */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return NULL; + } + + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); + return NULL; + } + + /* The dbname must be specified in primary_conninfo for slot-sync to work */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not " + "specified in primary_conninfo.")); + return NULL; + } + + wrconn = walrcv_connect(PrimaryConnInfo, false, false, + "Logical Replication Launcher", &err); + if (!wrconn) + { + ereport(ERROR, + (errmsg("could not connect to the primary server: %s for " + "slots synchronization", err))); + + /* Try connecting again in next attempt */ + *retry = true; + } + + ereport(LOG, + (errmsg("connection established to primary for slots synchronization"))); + + return wrconn; +} + +/* + * Re-read the config file. + * + * If one of the slot-sync options has changed, stop the slot-sync + * workers and disconnect the connection to primary. + */ +static void +LauncherRereadConfig(WalReceiverConn **wrconn, bool *ss_retry) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot-sync workers. The + * workers will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (feedback != hot_standby_feedback)) + { + slotsync_workers_stop(); + + if (*wrconn) + { + walrcv_disconnect(*wrconn); + *wrconn = NULL; + } + + /* Retry the connection with new GUCs */ + *ss_retry = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot-sync background workers. + * + * Connect to the primary, to get the list of DBIDs for failover logical slots. + * Then launch slot-sync workers (limited by max_slotsync_workers) where the DBs + * are distributed equally among those workers. + */ +static void +LaunchSlotSyncWorkers(long *wait_time, WalReceiverConn *wrconn) +{ + List *slots_dbs; + ListCell *lc; + MemoryContext tmpctx; + MemoryContext oldctx; + + Assert(wrconn); + + /* Use temporary context for the slot list and worker info. */ + tmpctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher slot-sync ctx", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(tmpctx); + + slots_dbs = walrcv_get_dbinfo_for_failover_slots(wrconn); + + slotsync_remove_obsolete_dbs(slots_dbs); + + foreach(lc, slots_dbs) + { + WalRcvFailoverSlotsData *slot_data = lfirst(lc); + SlotSyncWorker *w; + + Assert(OidIsValid(slot_data->dboid)); + + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + w = slotsync_worker_find(slot_data->dboid); + LWLockRelease(SlotSyncWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If launch failed, adjust the wait_time to retry in the next + * sync-cycle sooner. + */ + if (!slotsync_worker_launch_or_reuse(slot_data->dboid)) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + break; + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(tmpctx); +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + bool ss_retry = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1992,30 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot-sync workers else + * launch apply workers. + */ + if (RecoveryInProgress()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); + if (wrconn == NULL && ss_retry) + wrconn = slotsync_remote_connect(&wait_time, &ss_retry); - if (w != NULL) - continue; /* worker is running already */ - - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + if (wrconn) + LaunchSlotSyncWorkers(&wait_time, wrconn); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +2030,8 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&wrconn, &ss_retry); + } /* Not reachable */ @@ -1260,7 +2062,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +2101,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..4354b906d4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -524,6 +524,18 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. + */ + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + errdetail("This slot is being synced from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..1cee99a761 --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1090 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot-sync workers on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot-sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS + * during a slot creation, if it still does not catch up, abort the wait. + * The ones for which wait is aborted will attempt the wait and sync in the + * next sync-cycle. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot) + +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from" + " the primary server: %s", remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; + } + + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again.", + remote_slot->name)); + pfree(cmd.data); + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old lsns. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots() +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server and belongs to one of the DBs passed in. + */ +List * +get_local_synced_slot_names(Oid *dbids, uint32 dbcount) +{ + List *localSyncedSlots = NIL; + + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + for (int j = 0; j < dbcount; j++) + { + /* + * Add it to output list if this belongs to one of the + * worker's dbs. + */ + if (s->data.database == dbids[j]) + { + localSyncedSlots = lappend(localSyncedSlots, s); + break; + } + } + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(Oid *dbids, List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* + * Get the list of local slots for dbids managed by this worker, so that + * those not on remote could be dropped. + */ + local_slot_list = get_local_synced_slot_names(dbids, + MySlotSyncWorker->dbcount); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Construct Slot Query + * + * It constructs the query using dbids array in order to get failover + * logical slots information from the primary server. + */ +static void +construct_slot_query(StringInfo s, Oid *dbids) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_SHARED)); + + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i' and database IN "); + + appendStringInfoChar(s, '('); + for (int i = 0; i < MySlotSyncWorker->dbcount; i++) + { + char *dbname; + + if (i != 0) + appendStringInfoChar(s, ','); + + dbname = get_database_name(dbids[i]); + appendStringInfo(s, "%s", + quote_literal_cstr(dbname)); + pfree(dbname); + } + appendStringInfoChar(s, ')'); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot-sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we dont want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would move" + " it backwards", remote_slot->name)); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server for the dbids + * managed by this worker and then updates the slots locally as per the info + * received. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(dsa_area *dsa, WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + Oid *dbids; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* + * No more writes to dbcount and dbids by launcher after this until we + * release this lock. + */ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* + * Check dbcount before starting to sync. There is a possibility that + * dbids managed by this worker are no longer valid due to change in + * logical failover slots on the primary server. In that case, launcher + * will make dbcount=0 and will send SIGINT to shutdown this worker. Thus + * check dbcount before we proceed further. + */ + if (!MySlotSyncWorker->dbcount) + { + LWLockRelease(SlotSyncWorkerLock); + + /* Return and handle the interrupts in main loop */ + return false; + } + + /* Get dbids from dsa */ + dbids = (Oid *) dsa_get_address(dsa, MySlotSyncWorker->dbids_dp); + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s, dbids); + + elog(DEBUG2, "slot-sync worker %d's query:%s \n", MySlotSyncWorker->slot, + s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info from the primary server: %s", + res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + remote_slot->invalidated = RS_INVAL_NONE; + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(dbids, remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot-sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + MySlotSyncWorker->last_update_time = now; + else if (TimestampDifferenceExceeds(MySlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + LWLockRelease(SlotSyncWorkerLock); + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot-sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect() +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot-sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot-sync worker %d is shutting" + " down on receiving SIGINT", MySlotSyncWorker->slot)); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + int worker_slot = DatumGetInt32(main_arg); + dsa_handle handle; + dsa_area *dsa; + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* + * Attach to the dynamic shared memory segment for the slot-sync worker + * and find its table of contents. + */ + memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsa_handle)); + dsa = dsa_attach(handle); + if (!dsa) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not map dynamic shared memory " + "segment for slot-sync worker"))); + + /* Primary initialization is complete. Now attach to our slot. */ + slotsync_worker_attach(worker_slot); + + ereport(LOG, + errmsg("replication slot-sync worker %d started", worker_slot)); + + before_shmem_exit(slotsync_worker_detach, PointerGetDatum(dsa)); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(dsa, wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot-sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 7036096653..3caebd51f4 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index b706046811..c910c9be96 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -77,12 +77,14 @@ Node *replication_parse_result; %token K_EXPORT_SNAPSHOT %token K_NOEXPORT_SNAPSHOT %token K_USE_SNAPSHOT +%token K_LIST_DBID_FOR_FAILOVER_SLOTS %type <node> command %type <node> base_backup start_replication start_logical_replication create_replication_slot drop_replication_slot alter_replication_slot identify_system read_replication_slot timeline_history show + list_dbid_for_failover_slots %type <list> generic_option_list %type <defelt> generic_option %type <uintval> opt_timeline @@ -117,6 +119,7 @@ command: | read_replication_slot | timeline_history | show + | list_dbid_for_failover_slots ; /* @@ -129,6 +132,16 @@ identify_system: } ; +/* + * LIST_DBID_FOR_FAILOVER_SLOTS + */ +list_dbid_for_failover_slots: + K_LIST_DBID_FOR_FAILOVER_SLOTS + { + $$ = (Node *) makeNode(ListDBForFailoverSlotsCmd); + } + ; + /* * READ_REPLICATION_SLOT %s */ diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 0b5ae23195..57ddc08dfc 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -129,6 +129,7 @@ ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } +LIST_DBID_FOR_FAILOVER_SLOTS { return K_LIST_DBID_FOR_FAILOVER_SLOTS; } LOGICAL { return K_LOGICAL; } SLOT { return K_SLOT; } TEMPORARY { return K_TEMPORARY; } @@ -306,6 +307,7 @@ replication_scanner_is_replication_command(void) case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: + case K_LIST_DBID_FOR_FAILOVER_SLOTS: /* Yes; push back the first token so we can parse later. */ repl_pushed_back_token = first_token; return true; diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index d9ec6cdf07..89d1243014 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -318,6 +318,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -647,12 +648,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 3565ca196f..5ea75d7561 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * It return RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index d494ddfaad..a5f5fccde9 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -473,6 +473,73 @@ IdentifySystem(void) end_tup_output(tstate); } +/* + * Handle the LIST_DBID_FOR_FAILOVER_SLOTS command. + * + * Return the list of database-ids for failover logical slots. + * The returned list has no duplicates. + */ +static void +ListFailoverSlotsDbids(void) +{ + DestReceiver *dest; + TupOutputState *tstate; + TupleDesc tupdesc; + List *database_oids_list = NIL; + + dest = CreateDestReceiver(DestRemoteSimple); + + /* Need a tuple descriptor representing a single column */ + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "database_oid", + INT8OID, -1, 0); + + /* Prepare for projection of tuples */ + tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual); + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (int slotno = 0; slotno < max_replication_slots; slotno++) + { + ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[slotno]; + Oid dboid; + bool failover_slot = false; + Datum values[1]; + bool nulls[1]; + + if (!slot->in_use) + continue; + + SpinLockAcquire(&slot->mutex); + + dboid = slot->data.database; + failover_slot = slot->data.failover; + + SpinLockRelease(&slot->mutex); + + if (!failover_slot || SlotIsPhysical(slot)) + continue; + + /* Skip this slot if the database OID is already in the list. */ + if (list_member_oid(database_oids_list, dboid)) + continue; + + /* Add the database OID to the list */ + database_oids_list = lappend_oid(database_oids_list, dboid); + + values[0] = Int64GetDatum(dboid); + nulls[0] = (dboid == InvalidOid); + + /* Send it to dest */ + do_tup_output(tstate, values, nulls); + } + LWLockRelease(ReplicationSlotControlLock); + + /* Clean up the list */ + list_free(database_oids_list); + + end_tup_output(tstate); +} + /* Handle READ_REPLICATION_SLOT command */ static void ReadReplicationSlot(ReadReplicationSlotCmd *cmd) @@ -1255,7 +1322,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* @@ -2104,6 +2171,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_ListDBForFailoverSlotsCmd: + cmdtag = "LIST_DBID_FOR_FAILOVER_SLOTS"; + set_ps_display(cmdtag); + ListFailoverSlotsDbids(); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f6e2ec82c1..4affc1c861 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot-sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot-sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b671615c39..3f3db12cf3 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + true, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL @@ -3518,6 +3530,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_slotsync_workers", + PGC_POSTMASTER, + REPLICATION_STANDBY, + gettext_noop("Maximum number of slot synchronization workers " + "on a standby."), + NULL, + }, + &max_slotsync_workers, + 2, 0, 50, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..d83647889d 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,8 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary +#max_slotsync_workers = 2 # maximum number of slot synchronization workers # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 51faf936cb..52e186c4ea 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11082,14 +11082,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..236b89d8d5 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,12 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT int max_slotsync_workers; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +34,7 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..261c560190 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,8 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbinfo_for_failover_slots_fn walrcv_get_dbinfo_for_failover_slots; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +455,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbinfo_for_failover_slots(conn) \ + WalReceiverFunctions->walrcv_get_dbinfo_for_failover_slots(conn) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..a0d39b7572 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,11 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - /* Time at which this worker was launched. */ TimestampTz launch_time; @@ -53,6 +52,16 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,36 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot-sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot-sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorker +{ + LogicalWorkerHeader hdr; + + /* The slot in worker pool to which slot-sync worker is attached */ + int slot; + + /* Count of dbids slot-sync worker manages */ + uint32 dbcount; + + /* DSA for dbids */ + dsa_area *dbids_dsa; + + /* dsa_pointer for dbids slot-sync worker manages */ + dsa_pointer dbids_dp; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorker; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +273,17 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorker *MySlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(int slot); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); +extern List *get_local_synced_slot_names(Oid *dbids, uint32 dbcount); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +373,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +387,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..d26ba41c4d 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +$offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..70bd65e2cf 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | on enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 89f34aacbc..d1132653d1 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1430,6 +1430,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1509,6 +1510,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2312,6 +2314,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2570,6 +2573,7 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker SlruCtl SlruCtlData SlruErrorCause @@ -3017,6 +3021,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.30.0.windows.2 ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-14 14:26 Drouvot, Bertrand <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 1 sibling, 3 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-14 14:26 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/13/23 2:57 PM, Zhijie Hou (Fujitsu) wrote: > On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand <[email protected]> wrote: >> Yeah good point, agree to just error out in all the case then (if we discard the >> sync_ reserved wording proposal, which seems to be the case as probably not >> worth the extra work). > > Thanks for the discussion! > > Here is the V33 patch set which includes the following changes: Thanks for working on it! > > 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * this only after failure, so when you promote, we still * finish replaying as much as we can from archive and * pg_wal before failover. + * + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary + * server to catch up. */ if (StandbyMode && CheckForStandbyTrigger()) { XLogShutdownWalRcv(); + slotsync_drop_initiated_slots(); return XLREAD_FAIL; } I had a closer look and it seems this is not located at the right place. Indeed, it's added here: switch (currentSource) { case XLOG_FROM_ARCHIVE: case XLOG_FROM_PG_WAL: While in our case we are in case XLOG_FROM_STREAM: So I think we should move slotsync_drop_initiated_slots() in the XLOG_FROM_STREAM case. Maybe before shutting down the sync slot worker? (the TODO item number 2 you mentioned up-thread) BTW in order to prevent any corner case, would'nt also be better to replace: + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. + */ + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) with something like: if ((RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || slot->data.sync_state == SYNCSLOT_STATE_INITIATED) to ensure slots in 'i' case can never be used? Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-15 11:51 shveta malik <[email protected]> parent: Drouvot, Bertrand <[email protected]> 2 siblings, 1 reply; 55+ messages in thread From: shveta malik @ 2023-11-15 11:51 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Tue, Nov 14, 2023 at 7:56 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 11/13/23 2:57 PM, Zhijie Hou (Fujitsu) wrote: > > On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand <[email protected]> wrote: > >> Yeah good point, agree to just error out in all the case then (if we discard the > >> sync_ reserved wording proposal, which seems to be the case as probably not > >> worth the extra work). > > > > Thanks for the discussion! > > > > Here is the V33 patch set which includes the following changes: > > Thanks for working on it! > > > > > 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. > > @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, > * this only after failure, so when you promote, we still > * finish replaying as much as we can from archive and > * pg_wal before failover. > + * > + * Drop the slots for which sync is initiated but not yet > + * completed i.e. they are still waiting for the primary > + * server to catch up. > */ > if (StandbyMode && CheckForStandbyTrigger()) > { > XLogShutdownWalRcv(); > + slotsync_drop_initiated_slots(); > return XLREAD_FAIL; > } > > I had a closer look and it seems this is not located at the right place. > > Indeed, it's added here: > > switch (currentSource) > { > case XLOG_FROM_ARCHIVE: > case XLOG_FROM_PG_WAL: > > While in our case we are in > > case XLOG_FROM_STREAM: > > So I think we should move slotsync_drop_initiated_slots() in the > XLOG_FROM_STREAM case. Maybe before shutting down the sync slot worker? > (the TODO item number 2 you mentioned up-thread) > > BTW in order to prevent any corner case, would'nt also be better to > > replace: > > + /* > + * Do not allow consumption of a "synchronized" slot until the standby > + * gets promoted. > + */ > + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) > > with something like: > > if ((RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || slot->data.sync_state == SYNCSLOT_STATE_INITIATED) > > to ensure slots in 'i' case can never be used? > > Regards, > > -- > Bertrand Drouvot > PostgreSQL Contributors Team > RDS Open Source Databases > Amazon Web Services: https://aws.amazon.com PFA v34. It has changed patch002 from multi workers to single worker design as per the discussion in [1] and [2]. Please note that the TODO list mentioned in [3] is still pending and will be implemented in next version. [1]: https://www.postgresql.org/message-id/CAA4eK1JzYoHu2r%3D%2BKwn%2BN4ZgVcWKtdX_yLSNyTqjdWGkr-q0iA%40ma... [2]: https://www.postgresql.org/message-id/e7b63103-2a8c-4ee9-866a-ddba45ead388%40gmail.com [3]: https://www.postgresql.org/message-id/OS0PR01MB5716CE0729CEB3B5994A954194B3A%40OS0PR01MB5716.jpnprd0... thanks Shveta Attachments: [application/octet-stream] v34-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (125.7K, ../../CAJpy0uAT-oGsgmb4dC=dYYRNY7WXEAAWGWU2kL5phLz3Wcdifg@mail.gmail.com/2-v34-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From 865bc359782b43d47c3b52d5c0aacf644a793166 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v34 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 3 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 61 +++- src/backend/replication/logical/worker.c | 40 ++- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 28 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 340 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/psql/describe.c | 9 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 2 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 43 files changed, 1223 insertions(+), 168 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fc35a46e5e..f4fe1995d9 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 93f068edcf..7fb55ae444 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27535,7 +27535,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27550,8 +27550,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 4206752881..4db796aa0b 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..9c595ca3c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..7036096653 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,32 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { + char buf[100]; + + buf[0] = '\0'; + + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + strcat(buf, "twophase"); + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + { + if (buf[0] != '\0') + strcat(buf, " and "); + strcat(buf, "failover"); + } + ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that %s can be enabled", + MySubscription->name, buf))); should_exit = true; } } @@ -1412,7 +1429,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1732,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1746,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1768,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 52a9f136ab..0313bc2035 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 781aa43cc4..d9ec6cdf07 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -649,6 +656,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2129,3 +2161,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..3565ca196f 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) if (startlsn < moveto) { - SpinLockAcquire(&MyReplicationSlot->mutex); - MyReplicationSlot->data.restart_lsn = moveto; - SpinLockRelease(&MyReplicationSlot->mutex); + PhysicalConfirmReceivedLocation(moveto); retlsn = moveto; - - /* - * Dirty the slot so as it is written out at the next checkpoint. Note - * that the LSN position advanced may still be lost in the event of a - * crash, but this makes the data consistent after a clean shutdown. - */ - ReplicationSlotMarkDirty(); } return retlsn; @@ -679,6 +678,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +734,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +774,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..d494ddfaad 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -260,7 +261,6 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch); static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); - /* Initialize walsender process before entering the main command loop */ void InitWalSender(void) @@ -974,12 +974,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1030,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1055,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1065,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1087,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1258,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1487,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1579,235 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Does this Wal Sender need to wake up logical walsender. + * + * Check if the physical slot of this walsender is specified in + * standby_slot_names GUC. + */ +static bool +WalSndWakeupNeeded() +{ + ListCell *lc; + List *standby_slots; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + return true; + } + + return false; +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndSlots(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Here logical walsender associated with failover logical slot waits + * for physical standbys corresponding to physical slots specified in + * standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndSlots(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to entering the loop in case we already know we have + * enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly + * interesting if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1828,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndSlots(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1843,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1882,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1930,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2097,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2030,7 +2315,7 @@ ProcessStandbyMessage(void) /* * Remember that a walreceiver just confirmed receipt of lsn `lsn`. */ -static void +void PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { bool changed = false; @@ -2049,6 +2334,9 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + + if (WalSndWakeupNeeded()) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); } /* @@ -2562,7 +2850,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3600,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3642,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..f6e2ec82c1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical standby confirmation in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..b671615c39 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "walsenders only after specified replication slots " + "confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 5077e7b358..11fe412169 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6563,7 +6563,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6622,10 +6623,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n" ", subpasswordrequired AS \"%s\"\n" - ", subrunasowner AS \"%s\"\n", + ", subrunasowner AS \"%s\"\n" + ", subfailoverstate AS \"%s\"\n", gettext_noop("Origin"), gettext_noop("Password required"), - gettext_noop("Run as owner?")); + gettext_noop("Run as owner?"), + gettext_noop("Enable failover?")); appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 93742fc6ac..5f065e5c55 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3302,7 +3302,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb58dee3bc..d906734750 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11100,17 +11100,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..ecbd3526c5 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..7655437510 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,8 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..0253752d03 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dba3498a13..bc057c74d8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3861,6 +3862,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.34.1 [application/octet-stream] v34-0002-Add-logical-slot-sync-capability-to-the-physical.patch (99.3K, ../../CAJpy0uAT-oGsgmb4dC=dYYRNY7WXEAAWGWU2kL5phLz3Wcdifg@mail.gmail.com/3-v34-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From 0eb95d0a9add82c0f954210822e97513f8f54a48 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v34 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker on the standby server ping the primary server at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. The replication launcher on the physical standby starts slot-sync worker which is then responsible to keep on syncing the logical failover slots from the primary server. The nap time of worker is tuned according to the activity on the primary. The worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. The logical slots created by slot-sync worker on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). Slots synced on the standby can be identified using 'sync_state' column of pg_replication_slots view. The values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for the remote slot on the primary server to catch up. 'r': ready for periodic syncs. --- doc/src/sgml/config.sgml | 29 +- doc/src/sgml/system-views.sgml | 17 + src/backend/access/transam/xlogrecovery.c | 6 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 43 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 595 ++++++++-- src/backend/replication/logical/logical.c | 12 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1008 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/slot.c | 17 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 2 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 7 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 27 + src/include/replication/worker_internal.h | 55 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 +++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 34 files changed, 1919 insertions(+), 153 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f4fe1995d9..838c3516d1 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,14 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4888,23 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..cc1adf9127 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,23 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..ce69917c2f 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -50,6 +50,7 @@ #include "postmaster/startup.h" #include "replication/slot.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * this only after failure, so when you promote, we still * finish replaying as much as we can from archive and * pg_wal before failover. + * + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary + * server to catch up. */ if (StandbyMode && CheckForStandbyTrigger()) { XLogShutdownWalRcv(); + slotsync_drop_initiated_slots(); return XLREAD_FAIL; } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 9c595ca3c9..0bc3264d20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..be9fdf40aa 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,7 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +102,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +416,44 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* + * If multiple dbnames are specified, then the last one will be + * returned + */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * @@ -998,7 +1039,7 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, */ static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, - bool failover) + bool failover) { StringInfoData cmd; PGresult *res; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..a774ef2e52 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -8,9 +8,12 @@ * src/backend/replication/logical/launcher.c * * NOTES - * This module contains the logical replication worker launcher which - * uses the background worker infrastructure to start the logical - * replication workers for every enabled subscription. + * This module contains the replication worker launcher which + * uses the background worker infrastructure to: + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode + * b) start the slot-sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. * *------------------------------------------------------------------------- */ @@ -22,6 +25,7 @@ #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "catalog/pg_authid.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -57,6 +61,9 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +bool enable_syncslot = true; + +SlotSyncWorkerInfo *SlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +77,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorkerInfo ss_worker; /* slot-sync worker */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +110,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +187,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slot-sync worker. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +197,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +212,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorkerInfo *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +280,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +308,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +370,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,7 +399,7 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && + if (w->hdr.in_use && !w->hdr.proc && TimestampDifferenceExceeds(w->launch_time, now, wal_receiver_timeout)) { @@ -438,9 +457,9 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +476,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +529,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +541,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical workers and slot-sync worker. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +573,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +587,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +614,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +627,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +646,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +694,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +723,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +745,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +754,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +763,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +798,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +823,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +960,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot-sync worker. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot-sync worker to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -1115,13 +1153,386 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot-sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorkerInfo * worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->last_update_time = 0; +} + +/* + * Attach slot-sync worker to SlotSyncWorkerInfo assigned by the launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach() { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker not initialized, " + "cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker is " + "already running, cannot attach"))); + } + + SlotSyncWorker->hdr.proc = MyProc; + + before_shmem_exit(slotsync_worker_detach, (Datum) 0); + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + SlotSyncWorker->hdr.in_use = false; + SlotSyncWorker->hdr.proc = NULL; + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Start slot-sync background worker. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch() +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + SlotSyncWorkerInfo *worker; + bool attach; + uint16 generation; + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + worker = SlotSyncWorker; + + /* Prepare the new worker. */ + worker->hdr.in_use = true; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* + * 'proc' will be assigned in ReplSlotSyncWorkerMain when the worker + * attaches to SlotSyncWorkerInfo. + */ + worker->hdr.proc = NULL; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + snprintf(bgw.bgw_name, BGW_MAXLEN, "replication slot-sync worker"); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot-sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + if (!attach) + ereport(WARNING, + (errmsg("replication slot-sync worker failed to attach"))); + + return attach; +} + +/* + * Internal function to stop the slot-sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorkerInfo * worker) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot-sync worker with pid: %d", + SlotSyncWorker->hdr.proc->pid))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Connect to the primary server for slotsync purpose and return the connection + * info. + */ +static bool +slotsync_checks(long *wait_time, bool *retry) +{ + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return false; + + /* + * Since the above GUC is set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo) are + * compatible with slot synchronization. If not, issue warnings. + */ + + /* The primary_slot_name is not set */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return false; + } + + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); + return false; + } + + /* The dbname must be specified in primary_conninfo for slot-sync to work */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not " + "specified in primary_conninfo.")); + return false; + } + + return true; +} + +/* + * Re-read the config file. + * + * If one of the slot-sync options has changed, stop the slot-sync worker + * and set ss_recheck flag to enable the caller to recheck slot-sync GUCs + * before restarting the worker + */ +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot-sync worker. The + * worker will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (feedback != hot_standby_feedback)) + { + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use) + slotsync_worker_stop_internal(SlotSyncWorker); + + /* Retry slot-sync with new GUCs */ + *ss_recheck = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot-sync background worker. + */ +static void +LaunchSlotSyncWorker(long *wait_time) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* The worker is running already */ + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use + && SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + return; + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If launch failed, adjust the wait_time to retry in the next sync-cycle + * sooner. + */ + if (!slotsync_worker_launch()) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + } +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + bool start_slotsync = false; + bool recheck_slotsync = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1550,32 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot-sync worker else + * launch apply workers. + */ + if (RecoveryInProgress()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; + /* Make validation checks first */ + if (recheck_slotsync) + start_slotsync = slotsync_checks(&wait_time, &recheck_slotsync); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); - - if (w != NULL) - continue; /* worker is running already */ - - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + /* Start slot-sync workers if checks passed */ + if (start_slotsync) + LaunchSlotSyncWorker(&wait_time); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +1590,8 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&recheck_slotsync); + } /* Not reachable */ @@ -1260,7 +1622,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +1661,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..4354b906d4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -524,6 +524,18 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. + */ + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + errdetail("This slot is being synced from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..391059544e --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1008 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot-sync worker on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot-sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS + * during a slot creation, if it still does not catch up, abort the wait. + * The ones for which wait is aborted will attempt the wait and sync in the + * next sync-cycle. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot) + +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from" + " the primary server: %s", remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; + } + + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again.", + remote_slot->name)); + pfree(cmd.data); + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old lsns. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots() +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server. + */ +static List * +get_local_synced_slot_names() +{ + List *localSyncedSlots = NIL; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + localSyncedSlots = lappend(localSyncedSlots, s); + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + /* + * Get the list of local 'synced' slot so that those not on remote could + * be dropped. + */ + local_slot_list = get_local_synced_slot_names(); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Constructs the query in order to get failover logical slots + * information from the primary server. + */ +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot-sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we don't want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would move" + " it backwards", remote_slot->name)); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server and update + * the slots locally. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot-sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info " + "from the primary server: %s", res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + remote_slot->invalidated = RS_INVAL_NONE; + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot-sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot-sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect() +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot-sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot-sync worker is shutting" + " down on receiving SIGINT")); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + slotsync_worker_attach(); + + ereport(LOG, errmsg("replication slot-sync worker started")); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot-sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 7036096653..3caebd51f4 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index d9ec6cdf07..89d1243014 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -318,6 +318,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -647,12 +648,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 3565ca196f..2487b57b20 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * Returns RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index d494ddfaad..6bb21ae66e 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1255,7 +1255,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f6e2ec82c1..4affc1c861 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot-sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot-sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b671615c39..cecd67188d 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + true, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..152e8bff64 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,7 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d906734750..6a8192ad83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11095,14 +11095,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..5649043bd4 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,11 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +33,7 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..9950e61a38 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,7 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +454,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..9fbf5aa873 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,14 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - - /* Time at which this worker was launched. */ - TimestampTz launch_time; - /* Indicates if this slot is used or free. */ bool in_use; @@ -53,6 +49,19 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* Time at which this worker was launched. */ + TimestampTz launch_time; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,24 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot-sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot-sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorkerInfo +{ + LogicalWorkerHeader hdr; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorkerInfo; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +261,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorkerInfo *SlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(void); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +360,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +374,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..e0909e5374 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +my $offset2 = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset2); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..70bd65e2cf 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | on enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bc057c74d8..388023b058 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1431,6 +1431,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1510,6 +1511,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2313,6 +2315,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2571,6 +2574,7 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker SlruCtl SlruCtlData SlruErrorCause @@ -3018,6 +3022,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.34.1 [application/octet-stream] v34-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch (7.9K, ../../CAJpy0uAT-oGsgmb4dC=dYYRNY7WXEAAWGWU2kL5phLz3Wcdifg@mail.gmail.com/4-v34-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch) download | inline diff: From e14e111ddbd29cb39300c5577890b446523d3a8b Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Wed, 15 Nov 2023 12:11:56 +0530 Subject: [PATCH v34 3/3] Allow slot-sync worker to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync worker. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 391059544e..ae2c417b2f 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -82,6 +82,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -546,6 +549,52 @@ construct_slot_query(StringInfo s) " WHERE failover and sync_state != 'i'"); } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Synchronize single slot to given position. * @@ -730,6 +779,7 @@ synchronize_slots(WalReceiverConn *wrconn) List *remote_slot_list = NIL; MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -779,6 +829,9 @@ synchronize_slots(WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for lsns and xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -825,6 +878,17 @@ synchronize_slots(WalReceiverConn *wrconn) */ drop_obsolete_slots(remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -882,12 +946,18 @@ remote_connect() * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndSlots(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -905,7 +975,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot-sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -921,7 +992,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -972,7 +1046,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 6bb21ae66e..c48a542913 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1611,7 +1611,7 @@ WalSndWakeupNeeded() * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndSlots(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1634,13 +1634,12 @@ WalSndRereadConfigAndSlots(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1707,10 +1706,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index ecbd3526c5..25c522993f 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndSlots(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.34.1 ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-16 05:13 shveta malik <[email protected]> parent: Drouvot, Bertrand <[email protected]> 2 siblings, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-16 05:13 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Tue, Nov 14, 2023 at 7:56 PM Drouvot, Bertrand <[email protected]> wrote: > > Hi, > > On 11/13/23 2:57 PM, Zhijie Hou (Fujitsu) wrote: > > On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand <[email protected]> wrote: > >> Yeah good point, agree to just error out in all the case then (if we discard the > >> sync_ reserved wording proposal, which seems to be the case as probably not > >> worth the extra work). > > > > Thanks for the discussion! > > > > Here is the V33 patch set which includes the following changes: > > Thanks for working on it! > > > > > 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. > > @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, > * this only after failure, so when you promote, we still > * finish replaying as much as we can from archive and > * pg_wal before failover. > + * > + * Drop the slots for which sync is initiated but not yet > + * completed i.e. they are still waiting for the primary > + * server to catch up. > */ > if (StandbyMode && CheckForStandbyTrigger()) > { > XLogShutdownWalRcv(); > + slotsync_drop_initiated_slots(); > return XLREAD_FAIL; > } > > I had a closer look and it seems this is not located at the right place. > > Indeed, it's added here: > > switch (currentSource) > { > case XLOG_FROM_ARCHIVE: > case XLOG_FROM_PG_WAL: > > While in our case we are in > > case XLOG_FROM_STREAM: > > So I think we should move slotsync_drop_initiated_slots() in the > XLOG_FROM_STREAM case. Maybe before shutting down the sync slot worker? > (the TODO item number 2 you mentioned up-thread) > > BTW in order to prevent any corner case, would'nt also be better to > > replace: > > + /* > + * Do not allow consumption of a "synchronized" slot until the standby > + * gets promoted. > + */ > + if (RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) > > with something like: > > if ((RecoveryInProgress() && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || slot->data.sync_state == SYNCSLOT_STATE_INITIATED) > > to ensure slots in 'i' case can never be used? > Yes, it makes sense. WIll do it. > Regards, > > -- > Bertrand Drouvot > PostgreSQL Contributors Team > RDS Open Source Databases > Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-16 10:13 Amit Kapila <[email protected]> parent: shveta malik <[email protected]> 0 siblings, 2 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-16 10:13 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Wed, Nov 15, 2023 at 5:21 PM shveta malik <[email protected]> wrote: > > PFA v34. > Few comments on v34-0001* ======================= 1. + char buf[100]; + + buf[0] = '\0'; + + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + strcat(buf, "twophase"); + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + { + if (buf[0] != '\0') + strcat(buf, " and "); + strcat(buf, "failover"); + } + ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that %s can be enabled", + MySubscription->name, buf))); I feel it is better to separate elogs rather than construct the string. It would be easier for the translation. 2. - /* Initialize walsender process before entering the main command loop */ Spurious line removal 3. @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) if (startlsn < moveto) { - SpinLockAcquire(&MyReplicationSlot->mutex); - MyReplicationSlot->data.restart_lsn = moveto; - SpinLockRelease(&MyReplicationSlot->mutex); + PhysicalConfirmReceivedLocation(moveto); retlsn = moveto; - - /* - * Dirty the slot so as it is written out at the next checkpoint. Note - * that the LSN position advanced may still be lost in the event of a - * crash, but this makes the data consistent after a clean shutdown. - */ - ReplicationSlotMarkDirty(); } I think this change has been made so that we can wakeup logical walsenders from a central location. In general, this is a good idea but it seems calling PhysicalConfirmReceivedLocation() would make an additional call to ReplicationSlotsComputeRequiredLSN() which is already called in the caller of pg_physical_replication_slot_advance(), so not sure such unification is a good idea here. 4. + * Here logical walsender associated with failover logical slot waits + * for physical standbys corresponding to physical slots specified in + * standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) In the above comments, we don't seem to follow the 80-col limit. Please check all other comments in the patch for similar problem. 5. +static void +WalSndRereadConfigAndSlots(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} The function name is misleading w.r.t the functionality. Can we name it on the lines of WalSndRereadConfigAndReInitSlotList()? I know it is a bit longer but couldn't come up with anything better. 6. + /* + * Fast path to entering the loop in case we already know we have + * enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. I think this comment is a bit misleading because it is a fast path to avoid entering the loop. I think we can keep the existing comment here: "Fast path to avoid acquiring the spinlock in case we already know ..." 7. @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) The comment above this change needs to be updated for the usage of this new CV. 8. +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical standby confirmation in WAL sender process." I feel the above description is not clear. How about being more specific with something along the lines of: "Waiting for the WAL to be received by physical standby in WAL sender process." 9. + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), I think we slightly simplify it by saying: "Lists streaming replication standby server slot names that logical WAL sender processes wait for.". It would be more consistent with a few other similar variables. 10. + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "walsenders only after specified replication slots " + "confirm receiving WAL."), Instead of walsenders, let's use WAL sender processes. 11. @@ -6622,10 +6623,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n" ", subpasswordrequired AS \"%s\"\n" - ", subrunasowner AS \"%s\"\n", + ", subrunasowner AS \"%s\"\n" + ", subfailoverstate AS \"%s\"\n", gettext_noop("Origin"), gettext_noop("Password required"), - gettext_noop("Run as owner?")); + gettext_noop("Run as owner?"), + gettext_noop("Enable failover?")); Let's name the new column as "Failover" and also it should be displayed only when pset.sversion is >=17. 12. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ This should be listed in system_views.sql in the below GRANT statement: GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, subbinary, substream, subtwophasestate, subdisableonerr, subpasswordrequired, subrunasowner, subslotname, subsynccommit, subpublications, suborigin) 13. + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; It is better to add a comment for this new variable explaining its use. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-16 12:03 shveta malik <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 2 replies; 55+ messages in thread From: shveta malik @ 2023-11-16 12:03 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Thu, Nov 16, 2023 at 3:43 PM Amit Kapila <[email protected]> wrote: > > On Wed, Nov 15, 2023 at 5:21 PM shveta malik <[email protected]> wrote: > > > > PFA v34. > > > > Few comments on v34-0001* > ======================= > 1. > + char buf[100]; > + > + buf[0] = '\0'; > + > + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) > + strcat(buf, "twophase"); > + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) > + { > + if (buf[0] != '\0') > + strcat(buf, " and "); > + strcat(buf, "failover"); > + } > + > ereport(LOG, > - (errmsg("logical replication apply worker for subscription \"%s\" > will restart so that two_phase can be enabled", > - MySubscription->name))); > + (errmsg("logical replication apply worker for subscription \"%s\" > will restart so that %s can be enabled", > + MySubscription->name, buf))); > > I feel it is better to separate elogs rather than construct the > string. It would be easier for the translation. > > 2. > - > /* Initialize walsender process before entering the main command loop */ > > Spurious line removal > > 3. > @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) > > if (startlsn < moveto) > { > - SpinLockAcquire(&MyReplicationSlot->mutex); > - MyReplicationSlot->data.restart_lsn = moveto; > - SpinLockRelease(&MyReplicationSlot->mutex); > + PhysicalConfirmReceivedLocation(moveto); > retlsn = moveto; > - > - /* > - * Dirty the slot so as it is written out at the next checkpoint. Note > - * that the LSN position advanced may still be lost in the event of a > - * crash, but this makes the data consistent after a clean shutdown. > - */ > - ReplicationSlotMarkDirty(); > } > > I think this change has been made so that we can wakeup logical > walsenders from a central location. In general, this is a good idea > but it seems calling PhysicalConfirmReceivedLocation() would make an > additional call to ReplicationSlotsComputeRequiredLSN() which is > already called in the caller of > pg_physical_replication_slot_advance(), so not sure such unification > is a good idea here. > > 4. > + * Here logical walsender associated with failover logical slot waits > + * for physical standbys corresponding to physical slots specified in > + * standby_slot_names GUC. > + */ > +void > +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) > > In the above comments, we don't seem to follow the 80-col limit. > Please check all other comments in the patch for similar problem. > > 5. > +static void > +WalSndRereadConfigAndSlots(List **standby_slots) > +{ > + char *pre_standby_slot_names = pstrdup(standby_slot_names); > + > + ProcessConfigFile(PGC_SIGHUP); > + > + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) > + { > + list_free(*standby_slots); > + *standby_slots = GetStandbySlotList(true); > + } > + > + pfree(pre_standby_slot_names); > +} > > The function name is misleading w.r.t the functionality. Can we name > it on the lines of WalSndRereadConfigAndReInitSlotList()? I know it is > a bit longer but couldn't come up with anything better. > > 6. > + /* > + * Fast path to entering the loop in case we already know we have > + * enough WAL available and all the standby servers has confirmed > + * receipt of WAL upto RecentFlushPtr. > > I think this comment is a bit misleading because it is a fast path to > avoid entering the loop. I think we can keep the existing comment > here: "Fast path to avoid acquiring the spinlock in case we already > know ..." > > 7. > @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, > uint32 wait_event) > * And, we use separate shared memory CVs for physical and logical > * walsenders for selective wake ups, see WalSndWakeup() for more details. > */ > - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) > + if (wait_for_standby) > + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); > + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) > > The comment above this change needs to be updated for the usage of this new CV. > > 8. > +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical > standby confirmation in WAL sender process." > > I feel the above description is not clear. How about being more > specific with something along the lines of: "Waiting for the WAL to be > received by physical standby in WAL sender process." > > 9. > + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, > + gettext_noop("List of streaming replication standby server slot " > + "names that logical walsenders waits for."), > > I think we slightly simplify it by saying: "Lists streaming > replication standby server slot names that logical WAL sender > processes wait for.". It would be more consistent with a few other > similar variables. > > 10. > + gettext_noop("List of streaming replication standby server slot " > + "names that logical walsenders waits for."), > + gettext_noop("Decoded changes are sent out to plugins by logical " > + "walsenders only after specified replication slots " > + "confirm receiving WAL."), > > Instead of walsenders, let's use WAL sender processes. > > 11. > @@ -6622,10 +6623,12 @@ describeSubscriptions(const char *pattern, bool verbose) > appendPQExpBuffer(&buf, > ", suborigin AS \"%s\"\n" > ", subpasswordrequired AS \"%s\"\n" > - ", subrunasowner AS \"%s\"\n", > + ", subrunasowner AS \"%s\"\n" > + ", subfailoverstate AS \"%s\"\n", > gettext_noop("Origin"), > gettext_noop("Password required"), > - gettext_noop("Run as owner?")); > + gettext_noop("Run as owner?"), > + gettext_noop("Enable failover?")); > > Let's name the new column as "Failover" and also it should be > displayed only when pset.sversion is >=17. > > 12. > @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) > BKI_SHARED_RELATION BKI_ROW > bool subrunasowner; /* True if replication should execute as the > * subscription owner */ > > + char subfailoverstate; /* Enable Failover State */ > > This should be listed in system_views.sql in the below GRANT statement: > GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, > subbinary, substream, subtwophasestate, subdisableonerr, > subpasswordrequired, subrunasowner, > subslotname, subsynccommit, subpublications, suborigin) > > 13. > + ConditionVariable wal_confirm_rcv_cv; > + > WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; > } WalSndCtlData; > > It is better to add a comment for this new variable explaining its use. > > -- > With Regards, > Amit Kapila. PFA v35. It has below changes: 1) change of default for 'enable_syncslot' to false. 2) validate the dbname provided in primary_conninfo before attempting slot-sync. 3) do not allow logical decoding on slots with 'i' sync_state. 4) support in pg_upgrade for the failover property of slot. 5) do not start slot-sync if wal_level < logical 6) shutdown the slotsync worker on promotion. Thanks Ajin for working on 4 and 5. Thanks Hou-San for working on 6. The changes are in patch001 and patch002. With above changes, comments in [1] and [2] are addressed TODO: 1) Comments in [3]. 2) Analyze if we need to consider supporting an upgrade of the slot's 'sync_state' property? [1]: https://www.postgresql.org/message-id/OS0PR01MB571652CCD42F1D08D5BD69D494B3A%40OS0PR01MB5716.jpnprd0... [2]: https://www.postgresql.org/message-id/46070646-9e09-4566-8a62-ae31a12a510c%40gmail.com [3]: https://www.postgresql.org/message-id/CAA4eK1J%3D-kPHS1eHNBtzOQHZ64j6WSgSYQZ3fH%3D2vfiwy_48AA%40mail... thanks Shveta Attachments: [application/octet-stream] v35-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch (7.9K, ../../CAJpy0uB_y_1akXUWM9WnSdQ-NVWTnHTpmNu6gJQ5tnyuAMTUOg@mail.gmail.com/2-v35-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch) download | inline diff: From d5e4468353a674b09d65a3222994a8ce8d77446e Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Wed, 15 Nov 2023 12:11:56 +0530 Subject: [PATCH v35 3/3] Allow slot-sync worker to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync worker. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 85c454b93c..d404a95749 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -83,6 +83,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -548,6 +551,52 @@ construct_slot_query(StringInfo s) " WHERE failover and sync_state != 'i'"); } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Synchronize single slot to given position. * @@ -732,6 +781,7 @@ synchronize_slots(WalReceiverConn *wrconn) List *remote_slot_list = NIL; MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -781,6 +831,9 @@ synchronize_slots(WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for lsns and xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -827,6 +880,17 @@ synchronize_slots(WalReceiverConn *wrconn) */ drop_obsolete_slots(remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -884,12 +948,18 @@ remote_connect() * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndSlots(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -907,7 +977,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot-sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -923,7 +994,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -985,7 +1059,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 6bb21ae66e..c48a542913 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1611,7 +1611,7 @@ WalSndWakeupNeeded() * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndSlots(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1634,13 +1634,12 @@ WalSndRereadConfigAndSlots(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1707,10 +1706,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index ecbd3526c5..25c522993f 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndSlots(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.34.1 [application/octet-stream] v35-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (130.1K, ../../CAJpy0uB_y_1akXUWM9WnSdQ-NVWTnHTpmNu6gJQ5tnyuAMTUOg@mail.gmail.com/3-v35-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From 23fbc9b25c47f4acbfc6285d85a1269a2705a53e Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v35 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 3 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 61 +++- src/backend/replication/logical/worker.c | 40 ++- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 28 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 340 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/pg_upgrade/info.c | 5 +- src/bin/pg_upgrade/pg_upgrade.c | 10 +- src/bin/pg_upgrade/pg_upgrade.h | 2 + src/bin/pg_upgrade/t/003_logical_slots.pl | 6 +- src/bin/psql/describe.c | 9 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 2 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 47 files changed, 1240 insertions(+), 174 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fc35a46e5e..f4fe1995d9 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 93f068edcf..7fb55ae444 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27535,7 +27535,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27550,8 +27550,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 4206752881..4db796aa0b 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..9c595ca3c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..7036096653 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,32 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { + char buf[100]; + + buf[0] = '\0'; + + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + strcat(buf, "twophase"); + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + { + if (buf[0] != '\0') + strcat(buf, " and "); + strcat(buf, "failover"); + } + ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that %s can be enabled", + MySubscription->name, buf))); should_exit = true; } } @@ -1412,7 +1429,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1732,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1746,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1768,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 21abf34ef7..4fdac7bc66 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 781aa43cc4..d9ec6cdf07 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -649,6 +656,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2129,3 +2161,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..3565ca196f 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) if (startlsn < moveto) { - SpinLockAcquire(&MyReplicationSlot->mutex); - MyReplicationSlot->data.restart_lsn = moveto; - SpinLockRelease(&MyReplicationSlot->mutex); + PhysicalConfirmReceivedLocation(moveto); retlsn = moveto; - - /* - * Dirty the slot so as it is written out at the next checkpoint. Note - * that the LSN position advanced may still be lost in the event of a - * crash, but this makes the data consistent after a clean shutdown. - */ - ReplicationSlotMarkDirty(); } return retlsn; @@ -679,6 +678,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +734,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +774,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..d494ddfaad 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -260,7 +261,6 @@ static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch); static void WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); - /* Initialize walsender process before entering the main command loop */ void InitWalSender(void) @@ -974,12 +974,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1030,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1055,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1065,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1087,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1258,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1487,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1579,235 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Does this Wal Sender need to wake up logical walsender. + * + * Check if the physical slot of this walsender is specified in + * standby_slot_names GUC. + */ +static bool +WalSndWakeupNeeded() +{ + ListCell *lc; + List *standby_slots; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + return true; + } + + return false; +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndSlots(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Here logical walsender associated with failover logical slot waits + * for physical standbys corresponding to physical slots specified in + * standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndSlots(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to entering the loop in case we already know we have + * enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly + * interesting if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1828,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndSlots(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1843,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1882,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1930,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2097,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2030,7 +2315,7 @@ ProcessStandbyMessage(void) /* * Remember that a walreceiver just confirmed receipt of lsn `lsn`. */ -static void +void PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { bool changed = false; @@ -2049,6 +2334,9 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + + if (WalSndWakeupNeeded()) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); } /* @@ -2562,7 +2850,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3600,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3642,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..f6e2ec82c1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical standby confirmation in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..b671615c39 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("List of streaming replication standby server slot " + "names that logical walsenders waits for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "walsenders only after specified replication slots " + "confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 4878aa22bf..5db7c06164 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -661,7 +661,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) * started and stopped several times causing any temporary slots to be * removed. */ - res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, " + res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, failover, " "%s as caught_up, conflicting as invalid " "FROM pg_catalog.pg_replication_slots " "WHERE slot_type = 'logical' AND " @@ -681,6 +681,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) int i_twophase; int i_caught_up; int i_invalid; + int i_failover; slotinfos = (LogicalSlotInfo *) pg_malloc(sizeof(LogicalSlotInfo) * num_slots); @@ -689,6 +690,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) i_twophase = PQfnumber(res, "two_phase"); i_caught_up = PQfnumber(res, "caught_up"); i_invalid = PQfnumber(res, "invalid"); + i_failover = PQfnumber(res, "failover"); for (int slotnum = 0; slotnum < num_slots; slotnum++) { @@ -699,6 +701,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) curr->two_phase = (strcmp(PQgetvalue(res, slotnum, i_twophase), "t") == 0); curr->caught_up = (strcmp(PQgetvalue(res, slotnum, i_caught_up), "t") == 0); curr->invalid = (strcmp(PQgetvalue(res, slotnum, i_invalid), "t") == 0); + curr->failover = (strcmp(PQgetvalue(res, slotnum, i_failover), "t") == 0); } } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 3960af4036..90bed8ef72 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -916,8 +916,14 @@ create_logical_replication_slots(void) appendStringLiteralConn(query, slot_info->slotname, conn); appendPQExpBuffer(query, ", "); appendStringLiteralConn(query, slot_info->plugin, conn); - appendPQExpBuffer(query, ", false, %s);", - slot_info->two_phase ? "true" : "false"); + + if (GET_MAJOR_VERSION(new_cluster.major_version) >= 1700) + appendPQExpBuffer(query, ", false, %s, %s);", + slot_info->two_phase ? "true" : "false", + slot_info->failover ? "true" : "false"); + else + appendPQExpBuffer(query, ", false, %s);", + slot_info->two_phase ? "true" : "false"); PQclear(executeQueryOrDie(conn, "%s", query->data)); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index a710f325de..d47e950b77 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -160,6 +160,8 @@ typedef struct bool two_phase; /* can the slot decode 2PC? */ bool caught_up; /* has the slot caught up to latest changes? */ bool invalid; /* if true, the slot is unusable */ + bool failover; /* is the slot designated to be synced + * to the physical standby */ } LogicalSlotInfo; typedef struct diff --git a/src/bin/pg_upgrade/t/003_logical_slots.pl b/src/bin/pg_upgrade/t/003_logical_slots.pl index 5b01cf8c40..0a1c467ed0 100644 --- a/src/bin/pg_upgrade/t/003_logical_slots.pl +++ b/src/bin/pg_upgrade/t/003_logical_slots.pl @@ -158,7 +158,7 @@ $sub->start; $sub->safe_psql( 'postgres', qq[ CREATE TABLE tbl (a int); - CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true') + CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true', failover = 'true') ]); $sub->wait_for_subscription_sync($oldpub, 'regress_sub'); @@ -172,8 +172,8 @@ command_ok([@pg_upgrade_cmd], 'run of pg_upgrade of old cluster'); # Check that the slot 'regress_sub' has migrated to the new cluster $newpub->start; my $result = $newpub->safe_psql('postgres', - "SELECT slot_name, two_phase FROM pg_replication_slots"); -is($result, qq(regress_sub|t), 'check the slot exists on new cluster'); + "SELECT slot_name, two_phase, failover FROM pg_replication_slots"); +is($result, qq(regress_sub|t|t), 'check the slot exists on new cluster'); # Update the connection my $new_connstr = $newpub->connstr . ' dbname=postgres'; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 5077e7b358..11fe412169 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6563,7 +6563,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6622,10 +6623,12 @@ describeSubscriptions(const char *pattern, bool verbose) appendPQExpBuffer(&buf, ", suborigin AS \"%s\"\n" ", subpasswordrequired AS \"%s\"\n" - ", subrunasowner AS \"%s\"\n", + ", subrunasowner AS \"%s\"\n" + ", subfailoverstate AS \"%s\"\n", gettext_noop("Origin"), gettext_noop("Password required"), - gettext_noop("Run as owner?")); + gettext_noop("Run as owner?"), + gettext_noop("Enable failover?")); appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 006e10f5d2..7634c86262 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3308,7 +3308,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb58dee3bc..d906734750 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11100,17 +11100,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..ecbd3526c5 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..7655437510 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,8 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..0253752d03 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Enable failover? | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+------------------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dba3498a13..bc057c74d8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3861,6 +3862,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.34.1 [application/octet-stream] v35-0002-Add-logical-slot-sync-capability-to-the-physical.patch (102.3K, ../../CAJpy0uB_y_1akXUWM9WnSdQ-NVWTnHTpmNu6gJQ5tnyuAMTUOg@mail.gmail.com/4-v35-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From e017cfe546008c68ad32806a6f1e24d4720dd3a7 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v35 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker on the standby server ping the primary server at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. The replication launcher on the physical standby starts slot-sync worker which is then responsible to keep on syncing the logical failover slots from the primary server. The nap time of worker is tuned according to the activity on the primary. The worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. The logical slots created by slot-sync worker on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). Slots synced on the standby can be identified using 'sync_state' column of pg_replication_slots view. The values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for the remote slot on the primary server to catch up. 'r': ready for periodic syncs. --- doc/src/sgml/config.sgml | 30 +- doc/src/sgml/system-views.sgml | 18 + src/backend/access/transam/xlogrecovery.c | 11 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 44 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 669 +++++++++-- src/backend/replication/logical/logical.c | 20 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1021 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/slot.c | 17 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 2 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 9 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 27 + src/include/replication/worker_internal.h | 55 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 34 files changed, 2023 insertions(+), 154 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f4fe1995d9..cc0eafbf67 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,14 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4888,24 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked + after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..8cdb4684fb 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,24 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, + not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..8ea6dc799a 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -49,7 +49,9 @@ #include "postmaster/bgwriter.h" #include "postmaster/startup.h" #include "replication/slot.h" +#include "replication/logicallauncher.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1435,6 +1437,15 @@ FinishWalRecovery(void) */ XLogShutdownWalRcv(); + /* + * Shutdown the slot sync workers to prevent potential conflicts between + * user processes and slotsync workers after a promotion. Additionally, + * drop any slots that have initiated but not yet completed the sync + * process. + */ + ShutDownSlotSync(); + slotsync_drop_initiated_slots(); + /* * We are now done reading the xlog from stream. Turn off streaming * recovery to force fetching the files (which would be required at end of diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 9c595ca3c9..0bc3264d20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..5f82d01e6d 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,7 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +102,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +416,45 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* + * If multiple dbnames are specified, then the last one will be + * returned + */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && + opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * @@ -998,7 +1040,7 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, */ static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, - bool failover) + bool failover) { StringInfoData cmd; PGresult *res; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..2488a623fa 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -8,20 +8,26 @@ * src/backend/replication/logical/launcher.c * * NOTES - * This module contains the logical replication worker launcher which - * uses the background worker infrastructure to start the logical - * replication workers for every enabled subscription. + * This module contains the replication worker launcher which + * uses the background worker infrastructure to: + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode + * b) start the slot-sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. * *------------------------------------------------------------------------- */ #include "postgres.h" - +#include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "access/xlogrecovery.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_database.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -44,6 +50,7 @@ #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/ps_status.h" @@ -57,6 +64,9 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +bool enable_syncslot = false; + +SlotSyncWorkerInfo *SlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +80,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorkerInfo ss_worker; /* slot-sync worker */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +113,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +190,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slot-sync worker. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +200,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +215,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorkerInfo *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +283,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +311,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +373,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,7 +402,7 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && + if (w->hdr.in_use && !w->hdr.proc && TimestampDifferenceExceeds(w->launch_time, now, wal_receiver_timeout)) { @@ -438,9 +460,9 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +479,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +532,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +544,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical workers and slot-sync worker. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +576,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +590,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +617,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +630,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +649,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +697,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +726,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +748,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +757,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +766,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +801,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +826,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +963,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot-sync worker. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot-sync worker to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -1115,13 +1156,455 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot-sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorkerInfo * worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->last_update_time = 0; +} + +/* + * Attach slot-sync worker to SlotSyncWorkerInfo assigned by the launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach() { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker not initialized, " + "cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker is " + "already running, cannot attach"))); + } + + SlotSyncWorker->hdr.proc = MyProc; + + before_shmem_exit(slotsync_worker_detach, (Datum) 0); + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + SlotSyncWorker->hdr.in_use = false; + SlotSyncWorker->hdr.proc = NULL; + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Start slot-sync background worker. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch() +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + SlotSyncWorkerInfo *worker; + bool attach; + uint16 generation; + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + worker = SlotSyncWorker; + + /* Prepare the new worker. */ + worker->hdr.in_use = true; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* + * 'proc' will be assigned in ReplSlotSyncWorkerMain when the worker + * attaches to SlotSyncWorkerInfo. + */ + worker->hdr.proc = NULL; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + snprintf(bgw.bgw_name, BGW_MAXLEN, "replication slot-sync worker"); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot-sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", + "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + if (!attach) + ereport(WARNING, + (errmsg("replication slot-sync worker failed to attach"))); + + return attach; +} + +/* + * Internal function to stop the slot-sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorkerInfo *worker) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot-sync worker with pid: %d", + worker->hdr.proc->pid))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Validate if db with given dbname exists. + * + * Can't use existing functions like 'get_database_oid' from dbcommands.c for + * validity purpose as they need db connection. + */ +static bool +validate_dbname(const char *dbname) +{ + HeapTuple tuple; + Relation relation; + SysScanDesc scan; + ScanKeyData key[1]; + bool valid; + + /* Start a transaction so we can access pg_database */ + StartTransactionCommand(); + + /* Form a scan key */ + ScanKeyInit(&key[0], Anum_pg_database_datname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(dbname)); + + /* No db connection, force heap scan */ + relation = table_open(DatabaseRelationId, AccessShareLock); + scan = systable_beginscan(relation, DatabaseNameIndexId, false, + NULL, 1, key); + + tuple = systable_getnext(scan); + + if (HeapTupleIsValid(tuple)) + valid = true; + else + valid = false; + + /* all done */ + systable_endscan(scan); + table_close(relation, AccessShareLock); + + CommitTransactionCommand(); + return valid; +} + +/* + * Checks if GUC are set appropriately before starting slot-sync worker + */ +static bool +slotsync_checks(long *wait_time, bool *retry) +{ + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return false; + + /* + * Since the above GUC is set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo, wal_level) + * are compatible with slot synchronization. If not, issue warnings. + */ + + /* The primary_slot_name is not set */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return false; + } + + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); + return false; + } + + /* The wal_level must be set to logical for slot-sync to work */ + if (wal_level < WAL_LEVEL_LOGICAL) + { + ereport(WARNING, + errmsg("skipping slots synchronisation as it requires " + "wal_level >= logical")); + return false; + } + + /* The dbname must be specified in primary_conninfo for slot-sync to work */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not " + "specified in primary_conninfo.")); + return false; + } + + if (!validate_dbname(dbname)) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname specified " + "in primary_conninfo is not a valid one.")); + return false; + } + + return true; +} + +/* + * Shut down the slot-sync worker. + */ +void +ShutDownSlotSync(void) +{ + if (LogicalRepCtx->ss_worker.hdr.in_use) + slotsync_worker_stop_internal(&LogicalRepCtx->ss_worker); +} + +/* + * Re-read the config file. + * + * If one of the slot-sync options has changed, stop the slot-sync worker + * and set ss_recheck flag to enable the caller to recheck slot-sync GUCs + * before restarting the worker + */ +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot-sync worker. The + * worker will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (feedback != hot_standby_feedback)) + { + ShutDownSlotSync(); + + /* Retry slot-sync with new GUCs */ + *ss_recheck = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot-sync background worker. + */ +static void +LaunchSlotSyncWorker(long *wait_time) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* The worker is running already */ + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use + && SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + return; + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If launch failed, adjust the wait_time to retry in the next sync-cycle + * sooner. + */ + if (!slotsync_worker_launch()) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + } +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= + wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + bool start_slotsync = false; + bool recheck_slotsync = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1622,32 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot-sync worker else + * launch apply workers. + */ + if (RecoveryInProgress() && !PromoteIsTriggered()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); - - if (w != NULL) - continue; /* worker is running already */ + /* Make validation checks first */ + if (recheck_slotsync) + start_slotsync = slotsync_checks(&wait_time, &recheck_slotsync); - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + /* Start slot-sync workers if checks passed */ + if (start_slotsync) + LaunchSlotSyncWorker(&wait_time); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +1662,8 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&recheck_slotsync); + } /* Not reachable */ @@ -1260,7 +1694,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +1733,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..6ab6d5afb7 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -504,6 +504,7 @@ CreateDecodingContext(XLogRecPtr start_lsn, LogicalDecodingContext *ctx; ReplicationSlot *slot; MemoryContext old_context; + bool in_recovery; /* shorter lines... */ slot = MyReplicationSlot; @@ -524,6 +525,25 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + in_recovery = RecoveryInProgress(); + + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. Also do not allow consumption of slots with sync_state + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be + * used. + */ + if ((in_recovery && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || + slot->data.sync_state == SYNCSLOT_STATE_INITIATED) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + in_recovery ? + errdetail("This slot is being synced from the primary server.") : + errdetail("This slot was not synced completely from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..85c454b93c --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1021 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot-sync worker on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlogrecovery.h" +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot-sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS + * during a slot creation, if it still does not catch up, abort the wait. + * The ones for which wait is aborted will attempt the wait and sync in the + * next sync-cycle. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot) + +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from the" + " primary server: %s", + remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; + } + + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again.", + remote_slot->name)); + pfree(cmd.data); + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old lsns. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots() +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server. + */ +static List * +get_local_synced_slot_names() +{ + List *localSyncedSlots = NIL; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + localSyncedSlots = lappend(localSyncedSlots, s); + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + /* + * Get the list of local 'synced' slot so that those not on remote could + * be dropped. + */ + local_slot_list = get_local_synced_slot_names(); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Constructs the query in order to get failover logical slots + * information from the primary server. + */ +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot-sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we don't want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would" + " move it backwards", remote_slot->name)); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server and update + * the slots locally. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot-sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info " + "from the primary server: %s", res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + remote_slot->invalidated = RS_INVAL_NONE; + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot-sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot-sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect() +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot-sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot-sync worker is shutting" + " down on receiving SIGINT")); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + slotsync_worker_attach(); + + /* + * If the standby has been promoted, skip the slot synchronization process. + * + * Although the startup process stops all the slot-sync workers on + * promotion, the launcher may not have realized the promotion and could + * start additional workers after that. Therefore, this check is still + * necessary to prevent these additional workers from running. + */ + if (PromoteIsTriggered()) + exit(0); + + ereport(LOG, errmsg("replication slot-sync worker started")); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot-sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 7036096653..3caebd51f4 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index d9ec6cdf07..89d1243014 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -318,6 +318,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -647,12 +648,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 3565ca196f..2487b57b20 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * Returns RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index d494ddfaad..6bb21ae66e 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1255,7 +1255,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f6e2ec82c1..4affc1c861 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot-sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot-sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b671615c39..5d7a20a6aa 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..152e8bff64 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,7 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d906734750..6a8192ad83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11095,14 +11095,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..02499a7e66 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,11 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +33,9 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern void ShutDownSlotSync(void); + +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..9950e61a38 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,7 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +454,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..9fbf5aa873 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,14 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - - /* Time at which this worker was launched. */ - TimestampTz launch_time; - /* Indicates if this slot is used or free. */ bool in_use; @@ -53,6 +49,19 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* Time at which this worker was launched. */ + TimestampTz launch_time; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,24 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot-sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot-sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorkerInfo +{ + LogicalWorkerHeader hdr; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorkerInfo; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +261,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorkerInfo *SlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(void); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +360,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +374,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..d26ba41c4d 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +$offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..aac83755de 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | off enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bc057c74d8..388023b058 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1431,6 +1431,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1510,6 +1511,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2313,6 +2315,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2571,6 +2574,7 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker SlruCtl SlruCtlData SlruErrorCause @@ -3018,6 +3022,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.34.1 ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-17 01:46 Zhijie Hou (Fujitsu) <[email protected]> parent: Drouvot, Bertrand <[email protected]> 2 siblings, 1 reply; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-17 01:46 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand <[email protected]> wrote: > On 11/13/23 2:57 PM, Zhijie Hou (Fujitsu) wrote: > > On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand > <[email protected]> wrote: > >> Yeah good point, agree to just error out in all the case then (if we > >> discard the sync_ reserved wording proposal, which seems to be the > >> case as probably not worth the extra work). > > > > Thanks for the discussion! > > > > Here is the V33 patch set which includes the following changes: > > Thanks for working on it! > > > > > 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. > > @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr > RecPtr, bool randAccess, > * this only after failure, so when you promote, we still > * finish replaying as much as we can from archive and > * pg_wal before failover. > + * > + * Drop the slots for which sync is initiated but not yet > + * completed i.e. they are still waiting for the primary > + * server to catch up. > */ > if (StandbyMode && CheckForStandbyTrigger()) > { > XLogShutdownWalRcv(); > + slotsync_drop_initiated_slots(); > return XLREAD_FAIL; > } > > I had a closer look and it seems this is not located at the right place. > > Indeed, it's added here: > > switch (currentSource) > { > case XLOG_FROM_ARCHIVE: > case XLOG_FROM_PG_WAL: > > While in our case we are in > > case XLOG_FROM_STREAM: > > So I think we should move slotsync_drop_initiated_slots() in the > XLOG_FROM_STREAM case. Maybe before shutting down the sync slot worker? > (the TODO item number 2 you mentioned up-thread) Thanks for the comment. I feel the WaitForWALToBecomeAvailable may not be the best place to shutdown slotsync worker and drop slots. There could be other reasons(other than promotion) as mentioned in comments in case XLOG_FROM_STREAM to reach the code there. I thought if the intention is to stop slotsync workers on promotion, maybe FinishWalRecovery() is a better place to do it as it's indicating the end of recovery and XLogShutdownWalRcv is also called in it. And I feel we'd better drop the slots after shutting down the slotsync workers, because otherwise the slotsync workers could create the dropped slot again in rare cases. Best Regards, Hou zj ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-17 04:20 Ajin Cherian <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: Ajin Cherian @ 2023-11-17 04:20 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Amit Kapila <[email protected]>; shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Alvaro Herrera <[email protected]> On Tue, Nov 14, 2023 at 12:57 AM Zhijie Hou (Fujitsu) <[email protected]> wrote: > > 2) Raise error if user slot with same name already exists on standby. "ERROR: not synchronizing slot test; it is a user created slot" I just tested this using v35 and to me the error message when this happens is not very good. Neither does it sound like an error, nor is there clarity on what the underlying problem is or how to correct it. regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-17 11:38 Drouvot, Bertrand <[email protected]> parent: shveta malik <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-17 11:38 UTC (permalink / raw) To: shveta malik <[email protected]>; Amit Kapila <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/16/23 1:03 PM, shveta malik wrote: > On Thu, Nov 16, 2023 at 3:43 PM Amit Kapila <[email protected]> wrote: > > PFA v35. It has below changes: Thanks for the update! > 6) shutdown the slotsync worker on promotion. + /* + * Shutdown the slot sync workers to prevent potential conflicts between + * user processes and slotsync workers after a promotion. Additionally, + * drop any slots that have initiated but not yet completed the sync + * process. + */ + ShutDownSlotSync(); + slotsync_drop_initiated_slots(); I think there is a corner case here. If there is promotion while slot creation is in progress (slot has just been created and is in 'i' state), then when we shutdown the sync slot worker in ShutDownSlotSync() we'll set slot->in_use = false in ReplicationSlotDropPtr(). Indeed, when we shut the sync worker down: (gdb) bt #0 ReplicationSlotDropPtr (slot=0x7f25af5c9bb0) at slot.c:734 #1 0x000056266c8106a7 in ReplicationSlotDropAcquired () at slot.c:725 #2 0x000056266c810170 in ReplicationSlotRelease () at slot.c:583 #3 0x000056266c80f420 in ReplicationSlotShmemExit (code=1, arg=0) at slot.c:189 #4 0x000056266c86213b in shmem_exit (code=1) at ipc.c:243 #5 0x000056266c861fdf in proc_exit_prepare (code=1) at ipc.c:198 #6 0x000056266c861f23 in proc_exit (code=1) at ipc.c:111 So later on, when we'll want to drop this slot in slotsync_drop_initiated_slots() we'll get things like: 2023-11-17 11:22:08.526 UTC [2195486] FATAL: replication slot "logical_slot4" does not exist Reason is that slotsync_drop_initiated_slots() does call SearchNamedReplicationSlot(): (gdb) bt #0 SearchNamedReplicationSlot (name=0x7f743f5c9ab8 "logical_slot4", need_lock=false) at slot.c:388 #1 0x0000556ef0974ec1 in ReplicationSlotAcquire (name=0x7f743f5c9ab8 "logical_slot4", nowait=true) at slot.c:484 #2 0x0000556ef09754e7 in ReplicationSlotDrop (name=0x7f743f5c9ab8 "logical_slot4", nowait=true, user_cmd=false) at slot.c:668 #3 0x0000556ef095f0a3 in slotsync_drop_initiated_slots () at slotsync.c:369 that returns a NULL slot if slot->in_use = false. One option could be to make sure slot->in_use = true before calling ReplicationSlotDrop() here? + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-17 11:38 Amit Kapila <[email protected]> parent: shveta malik <[email protected]> 1 sibling, 2 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-17 11:38 UTC (permalink / raw) To: shveta malik <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thu, Nov 16, 2023 at 5:34 PM shveta malik <[email protected]> wrote: > > PFA v35. > Review v35-0002* ============== 1. As quoted in the commit message, > If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). > I think this won't happen normally because of the physical slot and hot_standby_feedback but probably can occur in cases like if the user temporarily switches hot_standby_feedback from on to off. Are there any other reasons? I think we can mention the cases along with it as well at least for now. Additionally, I think this should be covered in code comments as well. 2. #include "postgres.h" - +#include "access/genam.h" Spurious line removal. 3. A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. Is there a reason to remove part of the earlier sentence "use <literal>replication</literal> as the database name"? 4. + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked + after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> I think you forgot to update the documentation for the default value of this variable. 5. + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode + * b) start the slot-sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. Either use a full stop after both lines or none of these. 6. +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); There shouldn't be space between * and the worker. 7. + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker not initialized, " + "cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker is " + "already running, cannot attach"))); + } Using slot-sync in the error messages looks a bit odd to me. Can we use "replication slot sync worker ..." in both these and other similar messages? I think it would be better if we don't split the messages into multiple lines in these cases as messages don't appear too long to me. 8. +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ I think the reference to DSM is leftover from the previous version of the patch. Can we change the above comments as per the new code? 9. +static bool +slotsync_worker_launch() { ... + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; We should do something about this TODO. As per my understanding, we don't need a generation number for the slot sync worker as we have one such worker but I guess the patch requires it because we are using existing logical replication worker infrastructure. This brings the question of whether we really need a separate SlotSyncWorkerInfo or if we can use existing LogicalRepWorker and distinguish it with LogicalRepWorkerType? I guess you didn't use it because most of the fields in LogicalRepWorker will be unused for slot sync worker. 10. + * Can't use existing functions like 'get_database_oid' from dbcommands.c for + * validity purpose as they need db connection. + */ +static bool +validate_dbname(const char *dbname) I don't know how important it is to validate the dbname before launching the sync slot worker because anyway after launching, it will give an error while initializing the connection if the dbname is invalid. But, if we think it is really required, did you consider using GetDatabaseTuple()? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-17 11:48 Drouvot, Bertrand <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 0 siblings, 1 reply; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-17 11:48 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/17/23 2:46 AM, Zhijie Hou (Fujitsu) wrote: > On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand <[email protected]> wrote: >> On 11/13/23 2:57 PM, Zhijie Hou (Fujitsu) wrote: >>> On Friday, November 10, 2023 4:16 PM Drouvot, Bertrand >> <[email protected]> wrote: >>>> Yeah good point, agree to just error out in all the case then (if we >>>> discard the sync_ reserved wording proposal, which seems to be the >>>> case as probably not worth the extra work). >>> >>> Thanks for the discussion! >>> >>> Here is the V33 patch set which includes the following changes: >> >> Thanks for working on it! >> >>> >>> 1) Drop slots with state 'i' in promotion flow after we shut down WalReceiver. >> >> @@ -3557,10 +3558,15 @@ WaitForWALToBecomeAvailable(XLogRecPtr >> RecPtr, bool randAccess, >> * this only after failure, so when you promote, we still >> * finish replaying as much as we can from archive and >> * pg_wal before failover. >> + * >> + * Drop the slots for which sync is initiated but not yet >> + * completed i.e. they are still waiting for the primary >> + * server to catch up. >> */ >> if (StandbyMode && CheckForStandbyTrigger()) >> { >> XLogShutdownWalRcv(); >> + slotsync_drop_initiated_slots(); >> return XLREAD_FAIL; >> } >> >> I had a closer look and it seems this is not located at the right place. >> >> Indeed, it's added here: >> >> switch (currentSource) >> { >> case XLOG_FROM_ARCHIVE: >> case XLOG_FROM_PG_WAL: >> >> While in our case we are in >> >> case XLOG_FROM_STREAM: >> >> So I think we should move slotsync_drop_initiated_slots() in the >> XLOG_FROM_STREAM case. Maybe before shutting down the sync slot worker? >> (the TODO item number 2 you mentioned up-thread) > > Thanks for the comment. > > I feel the WaitForWALToBecomeAvailable may not be the best place to shutdown > slotsync worker and drop slots. There could be other reasons(other than > promotion) as mentioned in comments in case XLOG_FROM_STREAM to reach the code > there. I thought if the intention is to stop slotsync workers on promotion, > maybe FinishWalRecovery() is a better place to do it as it's indicating the end > of recovery and XLogShutdownWalRcv is also called in it. I can see that slotsync_drop_initiated_slots() has been moved in FinishWalRecovery() in v35. That looks ok. > > And I feel we'd better drop the slots after shutting down the slotsync workers, > because otherwise the slotsync workers could create the dropped slot again in > rare cases. Yeah, agree and I can see that it's done that way in v35. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-18 07:21 Zhijie Hou (Fujitsu) <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-18 07:21 UTC (permalink / raw) To: Amit Kapila <[email protected]>; shveta malik <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Thursday, November 16, 2023 6:13 PM Amit Kapila <[email protected]> wrote: > > On Wed, Nov 15, 2023 at 5:21 PM shveta malik <[email protected]> > wrote: > > > > PFA v34. > > > > Few comments on v34-0001* > ======================= > 1. > + char buf[100]; > + > + buf[0] = '\0'; > + > + if (MySubscription->twophasestate == > + LOGICALREP_TWOPHASE_STATE_PENDING) > + strcat(buf, "twophase"); > + if (MySubscription->failoverstate == > + LOGICALREP_FAILOVER_STATE_PENDING) > + { > + if (buf[0] != '\0') > + strcat(buf, " and "); > + strcat(buf, "failover"); > + } > + > ereport(LOG, > - (errmsg("logical replication apply worker for subscription \"%s\" > will restart so that two_phase can be enabled", > - MySubscription->name))); > + (errmsg("logical replication apply worker for subscription \"%s\" > will restart so that %s can be enabled", > + MySubscription->name, buf))); > > I feel it is better to separate elogs rather than construct the string. It would be > easier for the translation. > > 2. > - > /* Initialize walsender process before entering the main command loop */ > > Spurious line removal > > 3. > @@ -440,17 +448,8 @@ pg_physical_replication_slot_advance(XLogRecPtr > moveto) > > if (startlsn < moveto) > { > - SpinLockAcquire(&MyReplicationSlot->mutex); > - MyReplicationSlot->data.restart_lsn = moveto; > - SpinLockRelease(&MyReplicationSlot->mutex); > + PhysicalConfirmReceivedLocation(moveto); > retlsn = moveto; > - > - /* > - * Dirty the slot so as it is written out at the next checkpoint. Note > - * that the LSN position advanced may still be lost in the event of a > - * crash, but this makes the data consistent after a clean shutdown. > - */ > - ReplicationSlotMarkDirty(); > } > > I think this change has been made so that we can wakeup logical walsenders > from a central location. In general, this is a good idea but it seems calling > PhysicalConfirmReceivedLocation() would make an additional call to > ReplicationSlotsComputeRequiredLSN() which is already called in the caller of > pg_physical_replication_slot_advance(), so not sure such unification is a good > idea here. > > 4. > + * Here logical walsender associated with failover logical slot waits > + * for physical standbys corresponding to physical slots specified in > + * standby_slot_names GUC. > + */ > +void > +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) > > In the above comments, we don't seem to follow the 80-col limit. > Please check all other comments in the patch for similar problem. > > 5. > +static void > +WalSndRereadConfigAndSlots(List **standby_slots) { > + char *pre_standby_slot_names = pstrdup(standby_slot_names); > + > + ProcessConfigFile(PGC_SIGHUP); > + > + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) { > + list_free(*standby_slots); *standby_slots = GetStandbySlotList(true); > + } > + > + pfree(pre_standby_slot_names); > +} > > The function name is misleading w.r.t the functionality. Can we name it on the > lines of WalSndRereadConfigAndReInitSlotList()? I know it is a bit longer but > couldn't come up with anything better. > > 6. > + /* > + * Fast path to entering the loop in case we already know we have > + * enough WAL available and all the standby servers has confirmed > + * receipt of WAL upto RecentFlushPtr. > > I think this comment is a bit misleading because it is a fast path to avoid > entering the loop. I think we can keep the existing comment > here: "Fast path to avoid acquiring the spinlock in case we already know ..." > > 7. > @@ -3381,7 +3673,9 @@ WalSndWait(uint32 socket_events, long timeout, > uint32 wait_event) > * And, we use separate shared memory CVs for physical and logical > * walsenders for selective wake ups, see WalSndWakeup() for more details. > */ > - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) > + if (wait_for_standby) > + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); > + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) > > The comment above this change needs to be updated for the usage of this new > CV. > > 8. > +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for physical > standby confirmation in WAL sender process." > > I feel the above description is not clear. How about being more specific with > something along the lines of: "Waiting for the WAL to be received by physical > standby in WAL sender process." > > 9. > + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, > + gettext_noop("List of streaming replication standby server slot " > + "names that logical walsenders waits for."), > > I think we slightly simplify it by saying: "Lists streaming replication standby > server slot names that logical WAL sender processes wait for.". It would be > more consistent with a few other similar variables. > > 10. > + gettext_noop("List of streaming replication standby server slot " > + "names that logical walsenders waits for."), gettext_noop("Decoded > + changes are sent out to plugins by logical " > + "walsenders only after specified replication slots " > + "confirm receiving WAL."), > > Instead of walsenders, let's use WAL sender processes. > > 11. > @@ -6622,10 +6623,12 @@ describeSubscriptions(const char *pattern, bool > verbose) > appendPQExpBuffer(&buf, > ", suborigin AS \"%s\"\n" > ", subpasswordrequired AS \"%s\"\n" > - ", subrunasowner AS \"%s\"\n", > + ", subrunasowner AS \"%s\"\n" > + ", subfailoverstate AS \"%s\"\n", > gettext_noop("Origin"), > gettext_noop("Password required"), > - gettext_noop("Run as owner?")); > + gettext_noop("Run as owner?"), > + gettext_noop("Enable failover?")); > > Let's name the new column as "Failover" and also it should be displayed only > when pset.sversion is >=17. > > 12. > @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) > BKI_SHARED_RELATION BKI_ROW > bool subrunasowner; /* True if replication should execute as the > * subscription owner */ > > + char subfailoverstate; /* Enable Failover State */ > > This should be listed in system_views.sql in the below GRANT statement: > GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, > subbinary, substream, subtwophasestate, subdisableonerr, > subpasswordrequired, subrunasowner, > subslotname, subsynccommit, subpublications, suborigin) > > 13. > + ConditionVariable wal_confirm_rcv_cv; > + > WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; > > It is better to add a comment for this new variable explaining its use. Thanks for the comments. Here is the new version patch set which addressed all above comments and the comment in [1]. [1] https://www.postgresql.org/message-id/1e0b2eb4-c977-482d-b16e-c52711c34d6c%40gmail.com Best Regards, Hou zj Attachments: [application/octet-stream] v36-0002-Add-logical-slot-sync-capability-to-the-physical.patch (102.3K, ../../OS0PR01MB57169D5B90FA6B981BD7226794B6A@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v36-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From 73c3bccaedd053bf2d21fd000a723bf50638b3fa Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v36 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker on the standby server ping the primary server at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. The replication launcher on the physical standby starts slot-sync worker which is then responsible to keep on syncing the logical failover slots from the primary server. The nap time of worker is tuned according to the activity on the primary. The worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. The logical slots created by slot-sync worker on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). Slots synced on the standby can be identified using 'sync_state' column of pg_replication_slots view. The values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for the remote slot on the primary server to catch up. 'r': ready for periodic syncs. --- doc/src/sgml/config.sgml | 30 +- doc/src/sgml/system-views.sgml | 18 + src/backend/access/transam/xlogrecovery.c | 11 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 44 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 669 +++++++++-- src/backend/replication/logical/logical.c | 20 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1021 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/slot.c | 17 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 2 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 9 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 27 + src/include/replication/worker_internal.h | 55 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 34 files changed, 2023 insertions(+), 154 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f4fe1995d9..cc0eafbf67 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,14 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use - <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + <filename>~/.pgpass</filename> file on the standby server. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4888,24 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked + after failover. + </para> + <para> + It is enabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..8cdb4684fb 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,24 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, + not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..8ea6dc799a 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -49,7 +49,9 @@ #include "postmaster/bgwriter.h" #include "postmaster/startup.h" #include "replication/slot.h" +#include "replication/logicallauncher.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1435,6 +1437,15 @@ FinishWalRecovery(void) */ XLogShutdownWalRcv(); + /* + * Shutdown the slot sync workers to prevent potential conflicts between + * user processes and slotsync workers after a promotion. Additionally, + * drop any slots that have initiated but not yet completed the sync + * process. + */ + ShutDownSlotSync(); + slotsync_drop_initiated_slots(); + /* * We are now done reading the xlog from stream. Turn off streaming * recovery to force fetching the files (which would be required at end of diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 798c8d705d..750bebc124 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..5f82d01e6d 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,7 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +102,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +416,45 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* + * If multiple dbnames are specified, then the last one will be + * returned + */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && + opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * @@ -998,7 +1040,7 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, */ static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, - bool failover) + bool failover) { StringInfoData cmd; PGresult *res; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..2488a623fa 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -8,20 +8,26 @@ * src/backend/replication/logical/launcher.c * * NOTES - * This module contains the logical replication worker launcher which - * uses the background worker infrastructure to start the logical - * replication workers for every enabled subscription. + * This module contains the replication worker launcher which + * uses the background worker infrastructure to: + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode + * b) start the slot-sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. * *------------------------------------------------------------------------- */ #include "postgres.h" - +#include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "access/xlogrecovery.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_database.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -44,6 +50,7 @@ #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/ps_status.h" @@ -57,6 +64,9 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +bool enable_syncslot = false; + +SlotSyncWorkerInfo *SlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +80,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorkerInfo ss_worker; /* slot-sync worker */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +113,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +190,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slot-sync worker. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +200,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +215,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorkerInfo *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +283,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +311,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +373,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,7 +402,7 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && + if (w->hdr.in_use && !w->hdr.proc && TimestampDifferenceExceeds(w->launch_time, now, wal_receiver_timeout)) { @@ -438,9 +460,9 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +479,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +532,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +544,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical workers and slot-sync worker. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +576,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +590,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +617,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +630,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +649,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +697,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +726,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +748,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +757,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +766,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +801,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +826,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +963,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot-sync worker. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot-sync worker to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -1115,13 +1156,455 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot-sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorkerInfo * worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->last_update_time = 0; +} + +/* + * Attach slot-sync worker to SlotSyncWorkerInfo assigned by the launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach() { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker not initialized, " + "cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot-sync worker is " + "already running, cannot attach"))); + } + + SlotSyncWorker->hdr.proc = MyProc; + + before_shmem_exit(slotsync_worker_detach, (Datum) 0); + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the worker from DSM and update 'proc' and 'in_use'. + * Logical replication launcher will come to know using these + * that the worker has shutdown. + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + SlotSyncWorker->hdr.in_use = false; + SlotSyncWorker->hdr.proc = NULL; + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Start slot-sync background worker. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch() +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + SlotSyncWorkerInfo *worker; + bool attach; + uint16 generation; + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + worker = SlotSyncWorker; + + /* Prepare the new worker. */ + worker->hdr.in_use = true; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* + * 'proc' will be assigned in ReplSlotSyncWorkerMain when the worker + * attaches to SlotSyncWorkerInfo. + */ + worker->hdr.proc = NULL; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + snprintf(bgw.bgw_name, BGW_MAXLEN, "replication slot-sync worker"); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot-sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", + "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + if (!attach) + ereport(WARNING, + (errmsg("replication slot-sync worker failed to attach"))); + + return attach; +} + +/* + * Internal function to stop the slot-sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorkerInfo *worker) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot-sync worker with pid: %d", + worker->hdr.proc->pid))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); + + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Validate if db with given dbname exists. + * + * Can't use existing functions like 'get_database_oid' from dbcommands.c for + * validity purpose as they need db connection. + */ +static bool +validate_dbname(const char *dbname) +{ + HeapTuple tuple; + Relation relation; + SysScanDesc scan; + ScanKeyData key[1]; + bool valid; + + /* Start a transaction so we can access pg_database */ + StartTransactionCommand(); + + /* Form a scan key */ + ScanKeyInit(&key[0], Anum_pg_database_datname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(dbname)); + + /* No db connection, force heap scan */ + relation = table_open(DatabaseRelationId, AccessShareLock); + scan = systable_beginscan(relation, DatabaseNameIndexId, false, + NULL, 1, key); + + tuple = systable_getnext(scan); + + if (HeapTupleIsValid(tuple)) + valid = true; + else + valid = false; + + /* all done */ + systable_endscan(scan); + table_close(relation, AccessShareLock); + + CommitTransactionCommand(); + return valid; +} + +/* + * Checks if GUC are set appropriately before starting slot-sync worker + */ +static bool +slotsync_checks(long *wait_time, bool *retry) +{ + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return false; + + /* + * Since the above GUC is set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo, wal_level) + * are compatible with slot synchronization. If not, issue warnings. + */ + + /* The primary_slot_name is not set */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return false; + } + + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); + return false; + } + + /* The wal_level must be set to logical for slot-sync to work */ + if (wal_level < WAL_LEVEL_LOGICAL) + { + ereport(WARNING, + errmsg("skipping slots synchronisation as it requires " + "wal_level >= logical")); + return false; + } + + /* The dbname must be specified in primary_conninfo for slot-sync to work */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not " + "specified in primary_conninfo.")); + return false; + } + + if (!validate_dbname(dbname)) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname specified " + "in primary_conninfo is not a valid one.")); + return false; + } + + return true; +} + +/* + * Shut down the slot-sync worker. + */ +void +ShutDownSlotSync(void) +{ + if (LogicalRepCtx->ss_worker.hdr.in_use) + slotsync_worker_stop_internal(&LogicalRepCtx->ss_worker); +} + +/* + * Re-read the config file. + * + * If one of the slot-sync options has changed, stop the slot-sync worker + * and set ss_recheck flag to enable the caller to recheck slot-sync GUCs + * before restarting the worker + */ +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot-sync worker. The + * worker will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (feedback != hot_standby_feedback)) + { + ShutDownSlotSync(); + + /* Retry slot-sync with new GUCs */ + *ss_recheck = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot-sync background worker. + */ +static void +LaunchSlotSyncWorker(long *wait_time) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* The worker is running already */ + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use + && SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + return; + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If launch failed, adjust the wait_time to retry in the next sync-cycle + * sooner. + */ + if (!slotsync_worker_launch()) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + } +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= + wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + bool start_slotsync = false; + bool recheck_slotsync = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1622,32 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot-sync worker else + * launch apply workers. + */ + if (RecoveryInProgress() && !PromoteIsTriggered()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); - - if (w != NULL) - continue; /* worker is running already */ + /* Make validation checks first */ + if (recheck_slotsync) + start_slotsync = slotsync_checks(&wait_time, &recheck_slotsync); - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + /* Start slot-sync workers if checks passed */ + if (start_slotsync) + LaunchSlotSyncWorker(&wait_time); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +1662,8 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&recheck_slotsync); + } /* Not reachable */ @@ -1260,7 +1694,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +1733,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..6ab6d5afb7 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -504,6 +504,7 @@ CreateDecodingContext(XLogRecPtr start_lsn, LogicalDecodingContext *ctx; ReplicationSlot *slot; MemoryContext old_context; + bool in_recovery; /* shorter lines... */ slot = MyReplicationSlot; @@ -524,6 +525,25 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + in_recovery = RecoveryInProgress(); + + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. Also do not allow consumption of slots with sync_state + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be + * used. + */ + if ((in_recovery && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || + slot->data.sync_state == SYNCSLOT_STATE_INITIATED) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + in_recovery ? + errdetail("This slot is being synced from the primary server.") : + errdetail("This slot was not synced completely from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..ce9e729bfa --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1021 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot-sync worker on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlogrecovery.h" +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot-sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS + * during a slot creation, if it still does not catch up, abort the wait. + * The ones for which wait is aborted will attempt the wait and sync in the + * next sync-cycle. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot) + +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from the" + " primary server: %s", + remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; + } + + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again.", + remote_slot->name)); + pfree(cmd.data); + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old lsns. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots(void) +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->in_use && s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server. + */ +static List * +get_local_synced_slot_names(void) +{ + List *localSyncedSlots = NIL; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + localSyncedSlots = lappend(localSyncedSlots, s); + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + /* + * Get the list of local 'synced' slot so that those not on remote could + * be dropped. + */ + local_slot_list = get_local_synced_slot_names(); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Constructs the query in order to get failover logical slots + * information from the primary server. + */ +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot-sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we don't want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would" + " move it backwards", remote_slot->name)); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update lsns of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server and update + * the slots locally. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot-sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info " + "from the primary server: %s", res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + remote_slot->invalidated = RS_INVAL_NONE; + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot-sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot-sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect(void) +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot-sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot-sync worker is shutting" + " down on receiving SIGINT")); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + slotsync_worker_attach(); + + /* + * If the standby has been promoted, skip the slot synchronization process. + * + * Although the startup process stops all the slot-sync workers on + * promotion, the launcher may not have realized the promotion and could + * start additional workers after that. Therefore, this check is still + * necessary to prevent these additional workers from running. + */ + if (PromoteIsTriggered()) + exit(0); + + ereport(LOG, errmsg("replication slot-sync worker started")); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot-sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 6508d89c0f..b5bc58a8b0 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index d9ec6cdf07..89d1243014 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -318,6 +318,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -647,12 +648,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index fb6e37d2c3..7b1e0c1552 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * Returns RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index fec16bfc3d..e5871e6935 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1256,7 +1256,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index ede94a1ede..b3d9a98a5e 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot-sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot-sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index be05790ff3..a61d6d84e0 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..152e8bff64 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,7 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d906734750..6a8192ad83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11095,14 +11095,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..02499a7e66 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,11 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +33,9 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern void ShutDownSlotSync(void); + +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..9950e61a38 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,7 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +454,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..9fbf5aa873 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,14 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - - /* Time at which this worker was launched. */ - TimestampTz launch_time; - /* Indicates if this slot is used or free. */ bool in_use; @@ -53,6 +49,19 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* Time at which this worker was launched. */ + TimestampTz launch_time; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,24 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot-sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot-sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorkerInfo +{ + LogicalWorkerHeader hdr; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorkerInfo; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +261,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorkerInfo *SlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(void); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +360,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +374,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..d26ba41c4d 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +$offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..aac83755de 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | off enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bc057c74d8..388023b058 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1431,6 +1431,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1510,6 +1511,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2313,6 +2315,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2571,6 +2574,7 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker SlruCtl SlruCtlData SlruErrorCause @@ -3018,6 +3022,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.30.0.windows.2 [application/octet-stream] v36-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch (8.0K, ../../OS0PR01MB57169D5B90FA6B981BD7226794B6A@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v36-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch) download | inline diff: From 27dfcd6a62d5732d27ca00ef2e97dfd2426c4015 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Wed, 15 Nov 2023 12:11:56 +0530 Subject: [PATCH v36 3/3] Allow slot-sync worker to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync worker. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index ce9e729bfa..312e75adf1 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -83,6 +83,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -548,6 +551,52 @@ construct_slot_query(StringInfo s) " WHERE failover and sync_state != 'i'"); } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Synchronize single slot to given position. * @@ -732,6 +781,7 @@ synchronize_slots(WalReceiverConn *wrconn) List *remote_slot_list = NIL; MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -781,6 +831,9 @@ synchronize_slots(WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for lsns and xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -827,6 +880,17 @@ synchronize_slots(WalReceiverConn *wrconn) */ drop_obsolete_slots(remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -884,12 +948,18 @@ remote_connect(void) * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndReInitSlotList(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -907,7 +977,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot-sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -923,7 +994,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -985,7 +1059,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e5871e6935..45786c483d 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1615,7 +1615,7 @@ PhysicalWakeupLogicalWalSnd(void) * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndReInitSlotList(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1638,13 +1638,12 @@ WalSndRereadConfigAndReInitSlotList(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1711,10 +1710,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 1fcc22a127..c5c4714788 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndReInitSlotList(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.30.0.windows.2 [application/octet-stream] v36-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (129.6K, ../../OS0PR01MB57169D5B90FA6B981BD7226794B6A@OS0PR01MB5716.jpnprd01.prod.outlook.com/4-v36-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From 05b1d25deb8b127ce95c74368edc409357d79cca Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v36 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 6 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 57 ++- src/backend/replication/logical/worker.c | 40 +- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 19 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 342 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/pg_upgrade/info.c | 5 +- src/bin/pg_upgrade/pg_upgrade.c | 10 +- src/bin/pg_upgrade/pg_upgrade.h | 2 + src/bin/pg_upgrade/t/003_logical_slots.pl | 6 +- src/bin/psql/describe.c | 8 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 7 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 47 files changed, 1248 insertions(+), 162 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fc35a46e5e..f4fe1995d9 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 93f068edcf..7fb55ae444 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27535,7 +27535,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27550,8 +27550,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 4206752881..4db796aa0b 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..798c8d705d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); @@ -1333,7 +1334,8 @@ REVOKE ALL ON pg_subscription FROM public; GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, subbinary, substream, subtwophasestate, subdisableonerr, subpasswordrequired, subrunasowner, - subslotname, subsynccommit, subpublications, suborigin) + subslotname, subsynccommit, subpublications, suborigin, + subfailoverstate) ON pg_subscription TO public; CREATE VIEW pg_stat_subscription_stats AS diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..6508d89c0f 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,26 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", + MySubscription->name))); + + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that failover can be enabled", + MySubscription->name))); + should_exit = true; } } @@ -1412,7 +1423,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1726,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1740,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1762,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 21abf34ef7..4fdac7bc66 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 781aa43cc4..d9ec6cdf07 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -649,6 +656,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2129,3 +2161,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..fb6e37d2c3 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -451,6 +459,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) * crash, but this makes the data consistent after a clean shutdown. */ ReplicationSlotMarkDirty(); + + PhysicalWakeupLogicalWalSnd(); } return retlsn; @@ -679,6 +689,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +745,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +785,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..fec16bfc3d 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -974,12 +975,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1031,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1056,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1066,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1088,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1259,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1488,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1580,238 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Wake up logical walsenders with failover-enabled slots if the physical slot + * of the current walsender is specified in standby_slot_names GUC. + */ +void +PhysicalWakeupLogicalWalSnd(void) +{ + ListCell *lc; + List *standby_slots; + bool slot_in_list = false; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + { + slot_in_list = true; + break; + } + } + + if (slot_in_list) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndReInitSlotList(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Used by logical decoding SQL functions that acquired slot with failover + * enabled. It waits for physical standbys corresponding to the physical slots + * specified in the standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndReInitSlotList(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to avoid acquiring the spinlock in case we already know we + * have enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly interesting + * if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1832,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndReInitSlotList(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1847,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1886,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1934,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2101,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2049,6 +2338,7 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + PhysicalWakeupLogicalWalSnd(); } /* @@ -2562,7 +2852,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3602,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3644,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3380,8 +3674,14 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. + * + * When the wait_for_standby flag is set to true, wait on another CV that + * is woken up by physical walsenders when the walreceiver has confirmed + * the receipt of LSN. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..ede94a1ede 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for the WAL to be received by physical standby in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..be05790ff3 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("Lists streaming replication standby server slot " + "names that logical WAL sender processes wait for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "WAL sender processes only after specified " + "replication slots confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 4878aa22bf..5db7c06164 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -661,7 +661,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) * started and stopped several times causing any temporary slots to be * removed. */ - res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, " + res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, failover, " "%s as caught_up, conflicting as invalid " "FROM pg_catalog.pg_replication_slots " "WHERE slot_type = 'logical' AND " @@ -681,6 +681,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) int i_twophase; int i_caught_up; int i_invalid; + int i_failover; slotinfos = (LogicalSlotInfo *) pg_malloc(sizeof(LogicalSlotInfo) * num_slots); @@ -689,6 +690,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) i_twophase = PQfnumber(res, "two_phase"); i_caught_up = PQfnumber(res, "caught_up"); i_invalid = PQfnumber(res, "invalid"); + i_failover = PQfnumber(res, "failover"); for (int slotnum = 0; slotnum < num_slots; slotnum++) { @@ -699,6 +701,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) curr->two_phase = (strcmp(PQgetvalue(res, slotnum, i_twophase), "t") == 0); curr->caught_up = (strcmp(PQgetvalue(res, slotnum, i_caught_up), "t") == 0); curr->invalid = (strcmp(PQgetvalue(res, slotnum, i_invalid), "t") == 0); + curr->failover = (strcmp(PQgetvalue(res, slotnum, i_failover), "t") == 0); } } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 3960af4036..90bed8ef72 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -916,8 +916,14 @@ create_logical_replication_slots(void) appendStringLiteralConn(query, slot_info->slotname, conn); appendPQExpBuffer(query, ", "); appendStringLiteralConn(query, slot_info->plugin, conn); - appendPQExpBuffer(query, ", false, %s);", - slot_info->two_phase ? "true" : "false"); + + if (GET_MAJOR_VERSION(new_cluster.major_version) >= 1700) + appendPQExpBuffer(query, ", false, %s, %s);", + slot_info->two_phase ? "true" : "false", + slot_info->failover ? "true" : "false"); + else + appendPQExpBuffer(query, ", false, %s);", + slot_info->two_phase ? "true" : "false"); PQclear(executeQueryOrDie(conn, "%s", query->data)); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index a710f325de..d47e950b77 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -160,6 +160,8 @@ typedef struct bool two_phase; /* can the slot decode 2PC? */ bool caught_up; /* has the slot caught up to latest changes? */ bool invalid; /* if true, the slot is unusable */ + bool failover; /* is the slot designated to be synced + * to the physical standby */ } LogicalSlotInfo; typedef struct diff --git a/src/bin/pg_upgrade/t/003_logical_slots.pl b/src/bin/pg_upgrade/t/003_logical_slots.pl index 5b01cf8c40..0a1c467ed0 100644 --- a/src/bin/pg_upgrade/t/003_logical_slots.pl +++ b/src/bin/pg_upgrade/t/003_logical_slots.pl @@ -158,7 +158,7 @@ $sub->start; $sub->safe_psql( 'postgres', qq[ CREATE TABLE tbl (a int); - CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true') + CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true', failover = 'true') ]); $sub->wait_for_subscription_sync($oldpub, 'regress_sub'); @@ -172,8 +172,8 @@ command_ok([@pg_upgrade_cmd], 'run of pg_upgrade of old cluster'); # Check that the slot 'regress_sub' has migrated to the new cluster $newpub->start; my $result = $newpub->safe_psql('postgres', - "SELECT slot_name, two_phase FROM pg_replication_slots"); -is($result, qq(regress_sub|t), 'check the slot exists on new cluster'); + "SELECT slot_name, two_phase, failover FROM pg_replication_slots"); +is($result, qq(regress_sub|t|t), 'check the slot exists on new cluster'); # Update the connection my $new_connstr = $newpub->connstr . ' dbname=postgres'; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 5077e7b358..36795b1085 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6563,7 +6563,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6627,6 +6628,11 @@ describeSubscriptions(const char *pattern, bool verbose) gettext_noop("Password required"), gettext_noop("Run as owner?")); + if (pset.sversion >= 170000) + appendPQExpBuffer(&buf, + ", subfailoverstate AS \"%s\"\n", + gettext_noop("Failover")); + appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" ", subconninfo AS \"%s\"\n", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 006e10f5d2..7634c86262 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3308,7 +3308,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb58dee3bc..d906734750 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11100,17 +11100,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..1fcc22a127 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..48c6a7a146 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,13 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + /* + * Used by physical walsenders holding slots specified in + * standby_slot_names to wake up logical walsenders holding + * failover-enabled slots when a walreceiver confirms the receipt of LSN. + */ + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..7c0ef94330 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dba3498a13..bc057c74d8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3861,6 +3862,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.30.0.windows.2 ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-18 10:45 Amit Kapila <[email protected]> parent: Drouvot, Bertrand <[email protected]> 0 siblings, 3 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-18 10:45 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand <[email protected]> wrote: > > On 11/17/23 2:46 AM, Zhijie Hou (Fujitsu) wrote: > > On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand <[email protected]> wrote: > > > > I feel the WaitForWALToBecomeAvailable may not be the best place to shutdown > > slotsync worker and drop slots. There could be other reasons(other than > > promotion) as mentioned in comments in case XLOG_FROM_STREAM to reach the code > > there. I thought if the intention is to stop slotsync workers on promotion, > > maybe FinishWalRecovery() is a better place to do it as it's indicating the end > > of recovery and XLogShutdownWalRcv is also called in it. > > I can see that slotsync_drop_initiated_slots() has been moved in FinishWalRecovery() > in v35. That looks ok. > > I was thinking what if we just ignore creating such slots (which require init state) in the first place? I think that can be time-consuming in some cases but it will reduce the complexity and we can always improve such cases later if we really encounter them in the real world. I am not very sure that added complexity is worth addressing this particular case, so I would like to know your and others' opinions. More Review for v35-0002* ============================ 1. + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name " + "is not set.")); There is no need to use a full stop at the end for WARNING messages and as previously mentioned, let's not split message lines in such cases. There are other messages in the patch with similar problems, please fix those as well. 2. +slotsync_checks() { ... ... + /* The hot_standby_feedback must be ON for slot-sync to work */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback " + "is off.")); This message has the same problem as mentioned in the previous comment. Additionally, I think either atop slotsync_checks or along with GUC check we should write comments as to why we expect these values to be set for slot sync to work. 3. + /* The worker is running already */ + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use + && SlotSyncWorker->hdr.proc) The spacing for both the &&'s has problems. You need a space after the first && and the second && should be in the prior line. 4. + LauncherRereadConfig(&recheck_slotsync); + } An empty line after LauncherRereadConfig() is not required. 5. +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool feedback = hot_standby_feedback; Can we change the variable name 'feedback' to 'standbyfeedback' to make it slightly more descriptive? 6. The logic to recheck the slot_sync related parameters in LauncherMain() is not very clear. IIUC, if after reload config any parameter is changed, we just seem to be checking the validity of the changed parameter but not restarting the slot sync worker, is that correct? If so, what if dbname is changed, don't we need to restart the slot-sync worker and re-initialize the connection; similarly slotname change also needs some thoughts. Also, if all the parameters are valid we seem to be re-launching the slot-sync worker without first stopping it which doesn't seem correct, am I missing something in this logic? 7. @@ -524,6 +525,25 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + in_recovery = RecoveryInProgress(); + + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. Also do not allow consumption of slots with sync_state + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be + * used. + */ + if ((in_recovery && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || + slot->data.sync_state == SYNCSLOT_STATE_INITIATED) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + in_recovery ? + errdetail("This slot is being synced from the primary server.") : + errdetail("This slot was not synced completely from the primary server."), + errhint("Specify another replication slot."))); + If we are planning to drop slots in state SYNCSLOT_STATE_INITIATED at the time of promotion, don't we need to just have an assert or elog(ERROR, .. for non-recovery cases as such cases won't be reachable? If so, I think we can separate out that case here. 8. wait_for_primary_slot_catchup() { ... + /* Check if this standby is promoted while we are waiting */ + if (!RecoveryInProgress()) + { + /* + * The remote slot didn't pass the locally reserved position at + * the time of local promotion, so it's not safe to use. + */ + ereport( + WARNING, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg( + "slot-sync wait for slot %s interrupted by promotion, " + "slot creation aborted", remote_slot->name))); + pfree(cmd.data); + return false; + } ... } Shouldn't this be an Assert because a slot-sync worker shouldn't exist for non-standby servers? 9. wait_for_primary_slot_catchup() { ... + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + return false; If the slot on primary disappears, shouldn't this part of the code somehow ensure to remove the slot on standby as well? If it is taken at some other point in time then at least we should write a comment here to state how it is taken care of. I think this comment also applies to a few other checks following this check. 10. + /* + * It is possible to get null values for lsns and xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); We can say LSN and Xmin in the above comment to make it easier to read/understand. 11. /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null, so assert if found null. + */ No need to explicitly say about assert, it is clear from the code. We can slightly change this comment to: "Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin are expected to be valid/non-null." 12. + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * But still persist it and attempt the wait and sync in next + * sync-cycle. + */ + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } I think the reason to persist in this case is because next time local restart_lsn can be ahead than the current location and it can take more time to create such a slot. We can probably mention the same in the comments. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-20 09:47 Drouvot, Bertrand <[email protected]> parent: Amit Kapila <[email protected]> 2 siblings, 0 replies; 55+ messages in thread From: Drouvot, Bertrand @ 2023-11-20 09:47 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> Hi, On 11/18/23 11:45 AM, Amit Kapila wrote: > On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand > <[email protected]> wrote: >> >> On 11/17/23 2:46 AM, Zhijie Hou (Fujitsu) wrote: >>> On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand <[email protected]> wrote: >>> >>> I feel the WaitForWALToBecomeAvailable may not be the best place to shutdown >>> slotsync worker and drop slots. There could be other reasons(other than >>> promotion) as mentioned in comments in case XLOG_FROM_STREAM to reach the code >>> there. I thought if the intention is to stop slotsync workers on promotion, >>> maybe FinishWalRecovery() is a better place to do it as it's indicating the end >>> of recovery and XLogShutdownWalRcv is also called in it. >> >> I can see that slotsync_drop_initiated_slots() has been moved in FinishWalRecovery() >> in v35. That looks ok. >>> > > I was thinking what if we just ignore creating such slots (which > require init state) in the first place? I think that can be > time-consuming in some cases but it will reduce the complexity and we > can always improve such cases later if we really encounter them in the > real world. I am not very sure that added complexity is worth > addressing this particular case, so I would like to know your and > others' opinions. > I'm not sure I understand your point. Are you saying that we should not create slots on the standby that are "currently" reported in a 'i' state? (so just keep the 'r' and 'n' states?) Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-20 10:58 Amit Kapila <[email protected]> parent: Amit Kapila <[email protected]> 2 siblings, 2 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-20 10:58 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Sat, Nov 18, 2023 at 4:15 PM Amit Kapila <[email protected]> wrote: > > On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand > <[email protected]> wrote: > > More Review for v35-0002* > ============================ > More review of v35-0002* ==================== 1. +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) The name of the function is a bit misleading because it checks the validity of the slot not only whether it exists in remote_list. Would it be better to name it as ValidateSyncSlot() or something along those lines? 2. +static long +synchronize_slots(WalReceiverConn *wrconn) { ... + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); ... + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); ... +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) { ... + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); Do we really need to query a second time to get the invalidation cause? Can we adjust the slot_query to get it in one round trip? I think this may not optimize much because the patch uses second round trip only for invalidated slots but still looks odd. So unless the query becomes too complicated, we should try to achive it one round trip. 3. +static long +synchronize_slots(WalReceiverConn *wrconn) +{ ... ... + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot-sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); It is okay to perform the above query execution outside the transaction context but I would like to know the reason for the same. Do we want to retain anything beyond the transaction context or is there some other reason to do this outside the transaction context? 4. +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} Why would the sync_state on the primary server be any valid value? I thought it was set only on physical standby. I think it is better to mention the reason for using the sync state and or failover flag in the above comments. The current comment doesn't seem of much use as it just states what is evident from the query. 5. * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + This should be elog(ERROR, ..). Normally, we use elog(ERROR, ...) for such unexpected cases. And, you don't need to explicitly mention the last sentence in the comment: "But to take care of any bug in that flow, we should retain this check.". 6. +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) { ... + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would" + " move it backwards", remote_slot->name)); I think here elevel should be LOG because user can't do much about this. Do we use ';' at other places in the message? But when can we hit this case? We can add some comments to state in which scenario this possible. OTOH, if this is sort of can't happen case and we have kept it to avoid any sort of inconsistency then we can probably use elog(ERROR, .. with approapriate LSN locations, so that later the problem could be debugged. 7. +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) { ... + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + ... Similar to one of the previous comments, it is not clear to me why the patch is doing a memory context switch here. Can we add a comment? 8. + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } Won't we need error_code in this error? Also, the message doesn't seem to follow the code's usual style. 9. +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) { ... + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot-sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + How and when will this init state (SYNCSLOT_STATE_INITIATED) persist to disk? 10. + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) Empty line between if/else if is not required. 11. +static WalReceiverConn * +remote_connect() +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); Let's use appname similar to what we do for "walreceiver" as shown below: /* Establish the connection to the primary for XLOG streaming */ wrconn = walrcv_connect(conninfo, false, false, cluster_name[0] ? cluster_name : "walreceiver", &err); if (!wrconn) ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not connect to the primary server: %s", err))); Some proposals for default appname "slotsynchronizer", "slotsync worker". Also, use the same error code as used by "walreceiver". 12. Do we need the handling of the slotsync worker in GetBackendTypeDesc()? Please check without that what value this patch displays for backend_type. 13. +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); I think we should exit the worker in this case and allow it to reconnect. See the similar handling in maybe_reread_subscription(). One effect of not doing is that the dbname patch has used in ReplSlotSyncWorkerMain() will become inconsistent. 14. +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ ... ... + /* + * If the standby has been promoted, skip the slot synchronization process. + * + * Although the startup process stops all the slot-sync workers on + * promotion, the launcher may not have realized the promotion and could + * start additional workers after that. Therefore, this check is still + * necessary to prevent these additional workers from running. + */ + if (PromoteIsTriggered()) + exit(0); ... ... + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot-sync woker on promotion of standby")); I think we should never reach this code in non-standby mode. It should elog(ERROR,.. Can you please explain why promotion handling is required here? 15. @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; ... ... + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; These are not used in the patch. 16. +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; ... +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; These structures don't seem to be used in the current version of the patch. 17. --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" ... ... -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); Why the above two are removed as part of this patch? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-21 04:28 Amit Kapila <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: Amit Kapila @ 2023-11-21 04:28 UTC (permalink / raw) To: Drouvot, Bertrand <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Mon, Nov 20, 2023 at 4:28 PM Amit Kapila <[email protected]> wrote: > > 9. > +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, > + bool *slot_updated) > { > ... > + else > + { > + TransactionId xmin_horizon = InvalidTransactionId; > + ReplicationSlot *slot; > + > + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > + remote_slot->two_phase, false); > + slot = MyReplicationSlot; > + > + SpinLockAcquire(&slot->mutex); > + slot->data.database = get_database_oid(remote_slot->database, false); > + > + /* Mark it as sync initiated by slot-sync worker */ > + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; > + slot->data.failover = true; > + > + namestrcpy(&slot->data.plugin, remote_slot->plugin); > + SpinLockRelease(&slot->mutex); > + > + ReplicationSlotReserveWal(); > + > > How and when will this init state (SYNCSLOT_STATE_INITIATED) persist to disk? > On closer inspection, I see that it is done inside wait_for_primary_and_sync() when it fails to sync. I think it is better to refactor the code a bit and persist it in synchronize_one_slot() to make the code flow easier to understand. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-21 04:31 Zhijie Hou (Fujitsu) <[email protected]> parent: Amit Kapila <[email protected]> 2 siblings, 1 reply; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-21 04:31 UTC (permalink / raw) To: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; +Cc: shveta malik <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Saturday, November 18, 2023 6:46 PM Amit Kapila <[email protected]> wrote: > > On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand > <[email protected]> wrote: > > > > On 11/17/23 2:46 AM, Zhijie Hou (Fujitsu) wrote: > > > On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand > <[email protected]> wrote: > > > > > > I feel the WaitForWALToBecomeAvailable may not be the best place to > > > shutdown slotsync worker and drop slots. There could be other > > > reasons(other than > > > promotion) as mentioned in comments in case XLOG_FROM_STREAM to > > > reach the code there. I thought if the intention is to stop slotsync > > > workers on promotion, maybe FinishWalRecovery() is a better place to > > > do it as it's indicating the end of recovery and XLogShutdownWalRcv is also > called in it. > > > > I can see that slotsync_drop_initiated_slots() has been moved in > > FinishWalRecovery() in v35. That looks ok. > > > > > > More Review for v35-0002* Thanks for the comments. > ============================ > 1. > + ereport(WARNING, > + errmsg("skipping slots synchronization as primary_slot_name " > + "is not set.")); > > There is no need to use a full stop at the end for WARNING messages and as > previously mentioned, let's not split message lines in such cases. There are > other messages in the patch with similar problems, please fix those as well. Adjusted. > > 2. > +slotsync_checks() > { > ... > ... > + /* The hot_standby_feedback must be ON for slot-sync to work */ if > + (!hot_standby_feedback) { ereport(WARNING, errmsg("skipping slots > + synchronization as hot_standby_feedback " > + "is off.")); > > This message has the same problem as mentioned in the previous comment. > Additionally, I think either atop slotsync_checks or along with GUC check we > should write comments as to why we expect these values to be set for slot sync > to work. Added comments for these cases. > > 3. > + /* The worker is running already */ > + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use && > + SlotSyncWorker->hdr.proc) > > The spacing for both the &&'s has problems. You need a space after the first > && and the second && should be in the prior line. Adjusted. > > 4. > + LauncherRereadConfig(&recheck_slotsync); > + > } > > An empty line after LauncherRereadConfig() is not required. > > 5. > +static void > +LauncherRereadConfig(bool *ss_recheck) > +{ > + char *conninfo = pstrdup(PrimaryConnInfo); > + char *slotname = pstrdup(PrimarySlotName); > + bool syncslot = enable_syncslot; > + bool feedback = hot_standby_feedback; > > Can we change the variable name 'feedback' to 'standbyfeedback' to make it > slightly more descriptive? Changed. > > 6. The logic to recheck the slot_sync related parameters in > LauncherMain() is not very clear. IIUC, if after reload config any parameter is > changed, we just seem to be checking the validity of the changed parameter > but not restarting the slot sync worker, is that correct? If so, what if dbname is > changed, don't we need to restart the slot-sync worker and re-initialize the > connection; similarly slotname change also needs some thoughts. Also, if all the > parameters are valid we seem to be re-launching the slot-sync worker without > first stopping it which doesn't seem correct, am I missing something in this > logic? I think the slot sync worker will be stopped in LauncherRereadConfig() if GUC changed and new slot sync worker will be started in next loop in LauncherMain(). > 7. > @@ -524,6 +525,25 @@ CreateDecodingContext(XLogRecPtr start_lsn, > errmsg("replication slot \"%s\" was not created in this database", > NameStr(slot->data.name)))); > > + in_recovery = RecoveryInProgress(); > + > + /* > + * Do not allow consumption of a "synchronized" slot until the standby > + * gets promoted. Also do not allow consumption of slots with > + sync_state > + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be > + * used. > + */ > + if ((in_recovery && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || > + slot->data.sync_state == SYNCSLOT_STATE_INITIATED) > + ereport(ERROR, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg("cannot use replication slot \"%s\" for logical decoding", > + NameStr(slot->data.name)), in_recovery ? > + errdetail("This slot is being synced from the primary server.") : > + errdetail("This slot was not synced completely from the primary > + server."), errhint("Specify another replication slot."))); > + > > If we are planning to drop slots in state SYNCSLOT_STATE_INITIATED at the > time of promotion, don't we need to just have an assert or elog(ERROR, .. for > non-recovery cases as such cases won't be reachable? If so, I think we can > separate out that case here. Adjusted the codes as suggested. > > 8. > wait_for_primary_slot_catchup() > { > ... > + /* Check if this standby is promoted while we are waiting */ if > + (!RecoveryInProgress()) { > + /* > + * The remote slot didn't pass the locally reserved position at > + * the time of local promotion, so it's not safe to use. > + */ > + ereport( > + WARNING, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg( > + "slot-sync wait for slot %s interrupted by promotion, " > + "slot creation aborted", remote_slot->name))); pfree(cmd.data); return > + false; } > ... > } > > Shouldn't this be an Assert because a slot-sync worker shouldn't exist for > non-standby servers? Changed to Assert. > > 9. > wait_for_primary_slot_catchup() > { > ... > + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); > + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) { > + ereport(WARNING, (errmsg("slot \"%s\" disappeared from the primary > + server," > + " slot creation aborted", remote_slot->name))); pfree(cmd.data); > + walrcv_clear_result(res); return false; > > If the slot on primary disappears, shouldn't this part of the code somehow > ensure to remove the slot on standby as well? If it is taken at some other point > in time then at least we should write a comment here to state how it is taken > care of. I think this comment also applies to a few other checks following this > check. I adjusted the code here to not persist the slots if the slot disappeared or invalidated on primary, so that the local slot will get dropped when releasing. > > 10. > + /* > + * It is possible to get null values for lsns and xmin if slot is > + * invalidated on the primary server, so handle accordingly. > + */ > + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); > > We can say LSN and Xmin in the above comment to make it easier to > read/understand. Changed. > > 11. > /* > + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin > + * are expected to be valid/non-null, so assert if found null. > + */ > > No need to explicitly say about assert, it is clear from the code. We can slightly > change this comment to: "Once we got valid restart_lsn, then confirmed_lsn > and catalog_xmin are expected to be valid/non-null." Changed. > > 12. > + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || > + TransactionIdPrecedes(remote_slot->catalog_xmin, > + MyReplicationSlot->data.catalog_xmin)) > + { > + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) { > + /* > + * The remote slot didn't catch up to locally reserved position. > + * But still persist it and attempt the wait and sync in next > + * sync-cycle. > + */ > + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) { > + ReplicationSlotPersist(); *slot_updated = true; } > > I think the reason to persist in this case is because next time local restart_lsn can > be ahead than the current location and it can take more time to create such a > slot. We can probably mention the same in the comments. Updated the comments. Here is the V37 patch set which addressed comments above and [1]. [1] https://www.postgresql.org/message-id/CAA4eK1%2BP9R3GO2rwGBg2EOh%3DuYjWUSEOHD8yvs4Je8WYa2RHag%40mail... Best Regards, Hou zj Attachments: [application/octet-stream] v37-0002-Add-logical-slot-sync-capability-to-the-physical.patch (102.7K, ../../OS0PR01MB5716114A49E6DB9020766D6E94BBA@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v37-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From 761188988151f920e5539f1d716ba904a2c51c53 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v37 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker on the standby server ping the primary server at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. The replication launcher on the physical standby starts slot-sync worker which is then responsible to keep on syncing the logical failover slots from the primary server. The nap time of worker is tuned according to the activity on the primary. The worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. The logical slots created by slot-sync worker on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). Slots synced on the standby can be identified using 'sync_state' column of pg_replication_slots view. The values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for the remote slot on the primary server to catch up. 'r': ready for periodic syncs. --- doc/src/sgml/config.sgml | 29 +- doc/src/sgml/system-views.sgml | 18 + src/backend/access/transam/xlogrecovery.c | 11 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 44 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 667 +++++++++-- src/backend/replication/logical/logical.c | 22 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1032 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/slot.c | 18 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 2 +- src/backend/storage/lmgr/lwlock.c | 2 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 9 + src/include/replication/logicallauncher.h | 9 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 27 + src/include/replication/worker_internal.h | 55 +- src/include/storage/lwlock.h | 1 + src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 6 + 34 files changed, 2037 insertions(+), 152 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 69d68e8374..9dfe56f8fd 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,15 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use + <filename>~/.pgpass</filename> file on the standby server. (use <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4889,24 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked + after failover. + </para> + <para> + It is disabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..8cdb4684fb 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,24 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on physical + standby which has enabled slots synchronization. For the primary server, + its value is always 'none'. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but not yet completed, + not ready for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..8ea6dc799a 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -49,7 +49,9 @@ #include "postmaster/bgwriter.h" #include "postmaster/startup.h" #include "replication/slot.h" +#include "replication/logicallauncher.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1435,6 +1437,15 @@ FinishWalRecovery(void) */ XLogShutdownWalRcv(); + /* + * Shutdown the slot sync workers to prevent potential conflicts between + * user processes and slotsync workers after a promotion. Additionally, + * drop any slots that have initiated but not yet completed the sync + * process. + */ + ShutDownSlotSync(); + slotsync_drop_initiated_slots(); + /* * We are now done reading the xlog from stream. Turn off streaming * recovery to force fetching the files (which would be required at end of diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 798c8d705d..750bebc124 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..5f82d01e6d 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,7 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +102,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +416,45 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* + * If multiple dbnames are specified, then the last one will be + * returned + */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && + opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * @@ -998,7 +1040,7 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, */ static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, - bool failover) + bool failover) { StringInfoData cmd; PGresult *res; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..c0d6cf7e85 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -8,20 +8,27 @@ * src/backend/replication/logical/launcher.c * * NOTES - * This module contains the logical replication worker launcher which - * uses the background worker infrastructure to start the logical - * replication workers for every enabled subscription. + * This module contains the replication worker launcher which + * uses the background worker infrastructure to: + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode. + * b) start the slot sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. * *------------------------------------------------------------------------- */ #include "postgres.h" +#include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "access/xlogrecovery.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_database.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -44,6 +51,7 @@ #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/ps_status.h" @@ -57,6 +65,9 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +bool enable_syncslot = false; + +SlotSyncWorkerInfo *SlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +81,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorkerInfo ss_worker; /* slot sync worker */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +114,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorkerInfo *worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +191,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slot sync worker. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +201,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +216,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorkerInfo *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +284,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +312,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +374,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,7 +403,7 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && + if (w->hdr.in_use && !w->hdr.proc && TimestampDifferenceExceeds(w->launch_time, now, wal_receiver_timeout)) { @@ -438,9 +461,9 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +480,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +533,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +545,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical workers and slot sync worker. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +577,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +591,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +618,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +631,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +650,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +698,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +727,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +749,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +758,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +767,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +802,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +827,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +964,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot sync worker. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot sync worker to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -1115,13 +1157,455 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorkerInfo *worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->last_update_time = 0; +} + +/* + * Attach slot sync worker to SlotSyncWorkerInfo assigned by the launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach() { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot sync worker not initialized, cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot sync worker is already running, cannot attach"))); + } + + SlotSyncWorker->hdr.proc = MyProc; + + before_shmem_exit(slotsync_worker_detach, (Datum) 0); + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the slot sync worker (cleans up the worker info). + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(SlotSyncWorker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Start slot sync background worker. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch() +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + SlotSyncWorkerInfo *worker; + bool attach; + uint16 generation; + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + worker = SlotSyncWorker; + + /* Prepare the new worker. */ + worker->hdr.in_use = true; + + /* TODO: do we really need 'generation', analyse more here */ + worker->hdr.generation++; + + /* + * 'proc' will be assigned in ReplSlotSyncWorkerMain when the worker + * attaches to SlotSyncWorkerInfo. + */ + worker->hdr.proc = NULL; + + /* Before releasing lock, remember generation for future identification. */ + generation = worker->hdr.generation; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + snprintf(bgw.bgw_name, BGW_MAXLEN, "replication slot sync worker"); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + Assert(generation == worker->hdr.generation); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", + "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + SlotSyncWorkerLock); + + if (!attach) + ereport(WARNING, + (errmsg("replication slot sync worker failed to attach"))); + + return attach; +} + +/* + * Internal function to stop the slot sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorkerInfo *worker) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot sync worker with pid: %d", + worker->hdr.proc->pid))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Validate if db with given dbname exists. + * + * Can't use existing functions like 'get_database_oid' from dbcommands.c for + * validity purpose as they need db connection. + */ +static bool +validate_dbname(const char *dbname) +{ + HeapTuple tuple; + Relation relation; + SysScanDesc scan; + ScanKeyData key[1]; + bool valid; + + /* Start a transaction so we can access pg_database */ + StartTransactionCommand(); + + /* Form a scan key */ + ScanKeyInit(&key[0], Anum_pg_database_datname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(dbname)); + + /* No db connection, force heap scan */ + relation = table_open(DatabaseRelationId, AccessShareLock); + scan = systable_beginscan(relation, DatabaseNameIndexId, false, + NULL, 1, key); + + tuple = systable_getnext(scan); + + if (HeapTupleIsValid(tuple)) + valid = true; + else + valid = false; + + /* all done */ + systable_endscan(scan); + table_close(relation, AccessShareLock); + + CommitTransactionCommand(); + return valid; +} + +/* + * Checks if GUC are set appropriately before starting slot sync worker + */ +static bool +slotsync_checks(long *wait_time, bool *retry) +{ + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return false; + + /* + * Since the above GUC is set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo, wal_level) + * are compatible with slot synchronization. If not, issue warnings. + */ + + /* + * A physical replication slot(primary_slot_name) is required on the + * primary to ensure that the rows needed by the standby are not removed + * after restarting, so that the synchronized slot on the standby will not + * be invalidated. + */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name is not set")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return false; + } + + /* + * Hot_standby_feedback must be enabled to cooperate with the physical + * replication slot, which allows informing the primary about the xmin and + * catalog_xmin values on the standby. + */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback is off")); + return false; + } + + /* + * Logical decoding requires wal_level >= logical and we currently only + * synchronize logical slots. + */ + if (wal_level < WAL_LEVEL_LOGICAL) + { + ereport(WARNING, + errmsg("skipping slots synchronisation as it requires wal_level >= logical")); + return false; + } + + /* + * The slot sync worker needs a database connection for walrcv_exec to + * work. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not specified in primary_conninfo")); + return false; + } + + if (!validate_dbname(dbname)) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname specified in primary_conninfo is not a valid one")); + return false; + } + + return true; +} + +/* + * Shut down the slot sync worker. + */ +void +ShutDownSlotSync(void) +{ + if (LogicalRepCtx->ss_worker.hdr.in_use) + slotsync_worker_stop_internal(&LogicalRepCtx->ss_worker); +} + +/* + * Re-read the config file. + * + * If one of the slot sync options has changed, stop the slot sync worker + * and set ss_recheck flag to enable the caller to recheck slot sync GUCs + * before restarting the worker + */ +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool standbyfeedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot sync worker. The + * worker will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (standbyfeedback != hot_standby_feedback)) + { + ShutDownSlotSync(); + + /* Retry slot sync with new GUCs */ + *ss_recheck = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot sync background worker. + */ +static void +LaunchSlotSyncWorker(long *wait_time) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* The worker is running already */ + if (SlotSyncWorker && SlotSyncWorker->hdr.in_use && + SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + return; + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If launch failed, adjust the wait_time to retry in the next sync-cycle + * sooner. + */ + if (!slotsync_worker_launch()) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + } +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= + wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + bool start_slotsync = false; + bool recheck_slotsync = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1623,32 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot sync worker else + * launch apply workers. + */ + if (RecoveryInProgress() && !PromoteIsTriggered()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); + /* Make validation checks first */ + if (recheck_slotsync) + start_slotsync = slotsync_checks(&wait_time, &recheck_slotsync); - if (w != NULL) - continue; /* worker is running already */ - - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + /* Start slot sync workers if checks passed */ + if (start_slotsync) + LaunchSlotSyncWorker(&wait_time); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +1663,7 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&recheck_slotsync); } /* Not reachable */ @@ -1260,7 +1694,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +1733,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..5bef0138b4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -524,6 +524,28 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + /* + * Slots in state SYNCSLOT_STATE_INITIATED should have been dropped on + * promotion. + */ + if (!RecoveryInProgress() && slot->data.sync_state == SYNCSLOT_STATE_INITIATED) + elog(ERROR, "replication slot \"%s\" was not synced completely from the primary server", + NameStr(slot->data.name)); + + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. Also do not allow consumption of slots with sync_state + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be + * used. + */ + if (RecoveryInProgress() && slot->data.sync_state != SYNCSLOT_STATE_NONE) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + errdetail("This slot is being synced from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..61bbf42933 --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1032 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot sync worker on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlogrecovery.h" +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool conflicting; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for + * WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS during a slot creation, if it still + * does not catch up, abort the wait. The ones for which wait is aborted will + * attempt the wait and sync in the next sync-cycle. + * + * *persist will be set to false if the slot has disappeared or was invalidated + * on the primary; otherwise, it will be set to true. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *persist) +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + *persist = true; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + Assert(RecoveryInProgress()); + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from the" + " primary server: %s", + remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + + /* + * The slot being created will be dropped when it is released (see + * ReplicationSlotRelease). + */ + *persist = false; + + return false; + } + + + /* + * It is possible to get null values for LSN and Xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + + /* + * The slot being created will be dropped when it is released (see + * ReplicationSlotRelease). + */ + *persist = false; + + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again", + remote_slot->name)); + pfree(cmd.data); + + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Wait for remote slot to pass locally reserved position and + * sync the slot locally if wait is over. + */ +static void +wait_for_primary_and_sync(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of those on + * the remote then we cannot create the local slot in sync with the + * primary server because that would mean moving the local slot backwards + * and we might not have WALs retained for old LSN. In this case we will + * wait for the primary server's restart_lsn and catalog_xmin to catch up + * with the local one before attempting the sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + bool persist; + + if (!wait_for_primary_slot_catchup(wrconn, remote_slot, &persist)) + { + /* + * The remote slot didn't catch up to locally reserved position. + * + * We do not drop the slot because the restart_lsn can be ahead of + * the current location when recreating the slot in the next cycle. + * It may take more time to create such a slot. Therefore, we + * persist it and attempt the wait and synchronization in the next + * cycle. + */ + if (persist && MyReplicationSlot->data.persistency != RS_PERSISTENT) + { + ReplicationSlotPersist(); + *slot_updated = true; + } + + return; + } + } + + /* Update LSN of slot to remote slot's current position */ + local_slot_update(remote_slot); + + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) + ReplicationSlotPersist(); + else + ReplicationSlotSave(); + + /* + * Wait for primary is over, mark the slot as READY for incremental syncs. + * Cascading standbys can also start syncing it. + */ + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" has been synced", + remote_slot->name)); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots(void) +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->in_use && s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server. + */ +static List * +get_local_synced_slot_names(void) +{ + List *localSyncedSlots = NIL; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + localSyncedSlots = lappend(localSyncedSlots, s); + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * if remote slot is marked as non-conflicting (i.e. not + * invalidated) but local slot is marked as invalidated, then set + * the bool. + */ + *locally_invalidated = + !remote_slot->conflicting && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * This gets invalidation cause of the remote slot. + */ +static ReplicationSlotInvalidationCause +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) +{ + WalRcvExecResult *res; + Oid slotRow[1] = {INT2OID}; + StringInfoData cmd; + bool isnull; + TupleTableSlot *slot; + ReplicationSlotInvalidationCause cause; + MemoryContext oldctx = CurrentMemoryContext; + + /* Syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT pg_get_slot_invalidation_cause(%s)", + quote_literal_cstr(slot_name)); + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); + pfree(cmd.data); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch invalidation cause for slot \"%s\" from" + " the primary server: %s", slot_name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("slot \"%s\" disappeared from the primary server", + slot_name))); + + cause = DatumGetInt16(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + return cause; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + /* + * Get the list of local 'synced' slot so that those not on remote could + * be dropped. + */ + local_slot_list = get_local_synced_slot_names(); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = slot_exists_in_list(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Constructs the query in order to get failover logical slots + * information from the primary server. + */ +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch data for logical failover slots with sync_state either as + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, conflicting, " + " database FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + MemoryContext oldctx = CurrentMemoryContext; + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. But to + * take care of any bug in that flow, we should retain this check. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(LOG, "skipping sync of slot \"%s\" as the received slot sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* + * Already existing slot (created by slot sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we don't want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(WARNING, + errmsg("not synchronizing slot %s; synchronization would" + " move it backwards", remote_slot->name)); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update LSN of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to finish the first sync now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + errmsg("not synchronizing slot %s; it is a user created slot", + remote_slot->name)); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, false); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + /* Mark it as sync initiated by slot sync worker */ + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + slot->data.failover = true; + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + wait_for_primary_and_sync(wrconn, remote_slot, slot_updated); + + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server and update + * the slots locally. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 8 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* Make things live outside TX context */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info " + "from the primary server: %s", res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for LSN and Xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->conflicting = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + if (remote_slot->conflicting) + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, + remote_slot->name); + else + remote_slot->invalidated = RS_INVAL_NONE; + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect(void) +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot sync", &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Reconnect if GUC primary_conninfo got changed */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + *wrconn = remote_connect(); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot sync worker is shutting" + " down on receiving SIGINT")); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + slotsync_worker_attach(); + + /* + * If the standby has been promoted, skip the slot synchronization process. + * + * Although the startup process stops all the slot sync workers on + * promotion, the launcher may not have realized the promotion and could + * start additional workers after that. Therefore, this check is still + * necessary to prevent these additional workers from running. + */ + if (PromoteIsTriggered()) + exit(0); + + ereport(LOG, errmsg("replication slot sync worker started")); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, + NULL, + 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 6508d89c0f..b5bc58a8b0 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 85dd935e65..b903b90140 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -47,6 +47,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/proc.h" @@ -318,6 +319,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -677,12 +679,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index fb6e37d2c3..7b1e0c1552 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * Returns RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index fec16bfc3d..e5871e6935 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1256,7 +1256,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..fd9e73a49b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_SLOTSYNC_DSA: */ + "SlotSyncWorkerDSA", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index ede94a1ede..7eb735824c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index be05790ff3..a61d6d84e0 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..152e8bff64 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,7 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d906734750..6a8192ad83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11095,14 +11095,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..8a5b374cea 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,6 +33,15 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; +/* ------------------------------- + * LIST_DBID_FOR_FAILOVER_SLOTS command + * ------------------------------- + */ +typedef struct ListDBForFailoverSlotsCmd +{ + NodeTag type; + List *slot_names; +} ListDBForFailoverSlotsCmd; /* ---------------------- * BASE_BACKUP command diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..02499a7e66 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,11 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +33,9 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern void ShutDownSlotSync(void); + +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..9950e61a38 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -191,6 +192,14 @@ typedef struct } proto; } WalRcvStreamOptions; +/* + * Failover logical slots data received from remote. + */ +typedef struct WalRcvFailoverSlotsData +{ + Oid dboid; +} WalRcvFailoverSlotsData; + struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; @@ -280,6 +289,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +428,7 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +454,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..8843b9482f 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,14 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - - /* Time at which this worker was launched. */ - TimestampTz launch_time; - /* Indicates if this slot is used or free. */ bool in_use; @@ -53,6 +49,19 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* Time at which this worker was launched. */ + TimestampTz launch_time; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,24 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorkerInfo +{ + LogicalWorkerHeader hdr; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorkerInfo; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +261,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorkerInfo *SlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(void); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +360,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +374,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..0621ee70fc 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_SLOTSYNC_DSA, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..d26ba41c4d 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +$offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..aac83755de 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | off enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bc057c74d8..765e1e991a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1431,6 +1431,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1510,6 +1511,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2313,6 +2315,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2571,6 +2574,8 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker +SlotSyncWorkerInfo SlruCtl SlruCtlData SlruErrorCause @@ -3018,6 +3023,7 @@ WalLevel WalRcvData WalRcvExecResult WalRcvExecStatus +WalRcvFailoverSlotsData WalRcvState WalRcvStreamOptions WalRcvWakeupReason -- 2.30.0.windows.2 [application/octet-stream] v37-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch (8.0K, ../../OS0PR01MB5716114A49E6DB9020766D6E94BBA@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v37-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch) download | inline diff: From 8caf03f4890e55b9a7cfd9f99596bf64a71a5d4e Mon Sep 17 00:00:00 2001 From: Hou Zhijie <[email protected]> Date: Tue, 21 Nov 2023 12:12:23 +0800 Subject: [PATCH v37 3/3] Allow slot-sync worker to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync worker. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 61bbf42933..6f5b85d6c7 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -83,6 +83,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -559,6 +562,52 @@ construct_slot_query(StringInfo s) " WHERE failover and sync_state != 'i'"); } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Synchronize single slot to given position. * @@ -743,6 +792,7 @@ synchronize_slots(WalReceiverConn *wrconn) List *remote_slot_list = NIL; MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -792,6 +842,9 @@ synchronize_slots(WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for LSN and Xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -838,6 +891,17 @@ synchronize_slots(WalReceiverConn *wrconn) */ drop_obsolete_slots(remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -895,12 +959,18 @@ remote_connect(void) * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndReInitSlotList(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Reconnect if GUC primary_conninfo got changed */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -918,7 +988,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -934,7 +1005,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -996,7 +1070,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e5871e6935..45786c483d 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1615,7 +1615,7 @@ PhysicalWakeupLogicalWalSnd(void) * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndReInitSlotList(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1638,13 +1638,12 @@ WalSndRereadConfigAndReInitSlotList(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1711,10 +1710,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 1fcc22a127..c5c4714788 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndReInitSlotList(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.30.0.windows.2 [application/octet-stream] v37-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (129.6K, ../../OS0PR01MB5716114A49E6DB9020766D6E94BBA@OS0PR01MB5716.jpnprd01.prod.outlook.com/4-v37-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From ab503dbb04f327b6bc500fa2b5c6988e38340d80 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 31 Oct 2023 11:54:15 +0530 Subject: [PATCH v37 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 6 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 57 ++- src/backend/replication/logical/worker.c | 40 +- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 19 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 342 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/pg_upgrade/info.c | 5 +- src/bin/pg_upgrade/pg_upgrade.c | 10 +- src/bin/pg_upgrade/pg_upgrade.h | 2 + src/bin/pg_upgrade/t/003_logical_slots.pl | 6 +- src/bin/psql/describe.c | 8 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 7 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 47 files changed, 1248 insertions(+), 162 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 9398afbcbd..69d68e8374 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 93f068edcf..7fb55ae444 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27535,7 +27535,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27550,8 +27550,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 4206752881..4db796aa0b 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..798c8d705d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); @@ -1333,7 +1334,8 @@ REVOKE ALL ON pg_subscription FROM public; GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, subbinary, substream, subtwophasestate, subdisableonerr, subpasswordrequired, subrunasowner, - subslotname, subsynccommit, subpublications, suborigin) + subslotname, subsynccommit, subpublications, suborigin, + subfailoverstate) ON pg_subscription TO public; CREATE VIEW pg_stat_subscription_stats AS diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 37a0abe2f4..6508d89c0f 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,26 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", + MySubscription->name))); + + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that failover can be enabled", + MySubscription->name))); + should_exit = true; } } @@ -1412,7 +1423,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1714,10 +1726,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1725,9 +1740,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1741,9 +1762,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 21abf34ef7..4fdac7bc66 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 18bc28195b..85dd935e65 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -679,6 +686,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2159,3 +2191,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..fb6e37d2c3 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -451,6 +459,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) * crash, but this makes the data consistent after a clean shutdown. */ ReplicationSlotMarkDirty(); + + PhysicalWakeupLogicalWalSnd(); } return retlsn; @@ -679,6 +689,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +745,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +785,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e250b0567e..fec16bfc3d 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -974,12 +975,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1031,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1056,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1066,14 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); } else { @@ -1075,7 +1088,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); } if (cmd->kind == REPLICATION_KIND_LOGICAL) @@ -1246,6 +1259,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1488,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1580,238 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Wake up logical walsenders with failover-enabled slots if the physical slot + * of the current walsender is specified in standby_slot_names GUC. + */ +void +PhysicalWakeupLogicalWalSnd(void) +{ + ListCell *lc; + List *standby_slots; + bool slot_in_list = false; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + { + slot_in_list = true; + break; + } + } + + if (slot_in_list) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndReInitSlotList(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Used by logical decoding SQL functions that acquired slot with failover + * enabled. It waits for physical standbys corresponding to the physical slots + * specified in the standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndReInitSlotList(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to avoid acquiring the spinlock in case we already know we + * have enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly interesting + * if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1832,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndReInitSlotList(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1847,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1886,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1934,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2101,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2049,6 +2338,7 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + PhysicalWakeupLogicalWalSnd(); } /* @@ -2562,7 +2852,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3602,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3644,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3380,8 +3674,14 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. + * + * When the wait_for_standby flag is set to true, wait on another CV that + * is woken up by physical walsenders when the walreceiver has confirmed + * the receipt of LSN. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..ede94a1ede 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for the WAL to be received by physical standby in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..be05790ff3 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("Lists streaming replication standby server slot " + "names that logical WAL sender processes wait for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "WAL sender processes only after specified " + "replication slots confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 4878aa22bf..5db7c06164 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -661,7 +661,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) * started and stopped several times causing any temporary slots to be * removed. */ - res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, " + res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, failover, " "%s as caught_up, conflicting as invalid " "FROM pg_catalog.pg_replication_slots " "WHERE slot_type = 'logical' AND " @@ -681,6 +681,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) int i_twophase; int i_caught_up; int i_invalid; + int i_failover; slotinfos = (LogicalSlotInfo *) pg_malloc(sizeof(LogicalSlotInfo) * num_slots); @@ -689,6 +690,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) i_twophase = PQfnumber(res, "two_phase"); i_caught_up = PQfnumber(res, "caught_up"); i_invalid = PQfnumber(res, "invalid"); + i_failover = PQfnumber(res, "failover"); for (int slotnum = 0; slotnum < num_slots; slotnum++) { @@ -699,6 +701,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) curr->two_phase = (strcmp(PQgetvalue(res, slotnum, i_twophase), "t") == 0); curr->caught_up = (strcmp(PQgetvalue(res, slotnum, i_caught_up), "t") == 0); curr->invalid = (strcmp(PQgetvalue(res, slotnum, i_invalid), "t") == 0); + curr->failover = (strcmp(PQgetvalue(res, slotnum, i_failover), "t") == 0); } } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 3960af4036..90bed8ef72 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -916,8 +916,14 @@ create_logical_replication_slots(void) appendStringLiteralConn(query, slot_info->slotname, conn); appendPQExpBuffer(query, ", "); appendStringLiteralConn(query, slot_info->plugin, conn); - appendPQExpBuffer(query, ", false, %s);", - slot_info->two_phase ? "true" : "false"); + + if (GET_MAJOR_VERSION(new_cluster.major_version) >= 1700) + appendPQExpBuffer(query, ", false, %s, %s);", + slot_info->two_phase ? "true" : "false", + slot_info->failover ? "true" : "false"); + else + appendPQExpBuffer(query, ", false, %s);", + slot_info->two_phase ? "true" : "false"); PQclear(executeQueryOrDie(conn, "%s", query->data)); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index a710f325de..d47e950b77 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -160,6 +160,8 @@ typedef struct bool two_phase; /* can the slot decode 2PC? */ bool caught_up; /* has the slot caught up to latest changes? */ bool invalid; /* if true, the slot is unusable */ + bool failover; /* is the slot designated to be synced + * to the physical standby */ } LogicalSlotInfo; typedef struct diff --git a/src/bin/pg_upgrade/t/003_logical_slots.pl b/src/bin/pg_upgrade/t/003_logical_slots.pl index 5b01cf8c40..0a1c467ed0 100644 --- a/src/bin/pg_upgrade/t/003_logical_slots.pl +++ b/src/bin/pg_upgrade/t/003_logical_slots.pl @@ -158,7 +158,7 @@ $sub->start; $sub->safe_psql( 'postgres', qq[ CREATE TABLE tbl (a int); - CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true') + CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true', failover = 'true') ]); $sub->wait_for_subscription_sync($oldpub, 'regress_sub'); @@ -172,8 +172,8 @@ command_ok([@pg_upgrade_cmd], 'run of pg_upgrade of old cluster'); # Check that the slot 'regress_sub' has migrated to the new cluster $newpub->start; my $result = $newpub->safe_psql('postgres', - "SELECT slot_name, two_phase FROM pg_replication_slots"); -is($result, qq(regress_sub|t), 'check the slot exists on new cluster'); + "SELECT slot_name, two_phase, failover FROM pg_replication_slots"); +is($result, qq(regress_sub|t|t), 'check the slot exists on new cluster'); # Update the connection my $new_connstr = $newpub->connstr . ' dbname=postgres'; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 5077e7b358..36795b1085 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6563,7 +6563,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6627,6 +6628,11 @@ describeSubscriptions(const char *pattern, bool verbose) gettext_noop("Password required"), gettext_noop("Run as owner?")); + if (pset.sversion >= 170000) + appendPQExpBuffer(&buf, + ", subfailoverstate AS \"%s\"\n", + gettext_noop("Failover")); + appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" ", subconninfo AS \"%s\"\n", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 006e10f5d2..7634c86262 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3308,7 +3308,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb58dee3bc..d906734750 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11100,17 +11100,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..1fcc22a127 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..48c6a7a146 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,13 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + /* + * Used by physical walsenders holding slots specified in + * standby_slot_names to wake up logical walsenders holding + * failover-enabled slots when a walreceiver confirms the receipt of LSN. + */ + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..7c0ef94330 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dba3498a13..bc057c74d8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3861,6 +3862,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.30.0.windows.2 ^ permalink raw reply [nested|flat] 55+ messages in thread
* RE: Synchronizing slots from primary to standby @ 2023-11-21 04:31 Zhijie Hou (Fujitsu) <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 2 replies; 55+ messages in thread From: Zhijie Hou (Fujitsu) @ 2023-11-21 04:31 UTC (permalink / raw) To: Amit Kapila <[email protected]>; shveta malik <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]> On Friday, November 17, 2023 7:39 PM Amit Kapila <[email protected]> wrote: > > On Thu, Nov 16, 2023 at 5:34 PM shveta malik <[email protected]> > wrote: > > > > PFA v35. > > > > Review v35-0002* > ============== Thanks for the comments. > 1. > As quoted in the commit message, > > > If a logical slot is invalidated on the primary, slot on the standby is also > invalidated. If a logical slot on the primary is valid but is invalidated on the > standby due to conflict (say required rows removed on the primary), then that > slot is dropped and recreated on the standby in next sync-cycle. > It is okay to recreate such slots as long as these are not consumable on the > standby (which is the case currently). > > > > I think this won't happen normally because of the physical slot and > hot_standby_feedback but probably can occur in cases like if the user > temporarily switches hot_standby_feedback from on to off. Are there any other > reasons? I think we can mention the cases along with it as well at least for now. > Additionally, I think this should be covered in code comments as well. I will collect all these cases and update in next version. > > 2. > #include "postgres.h" > - > +#include "access/genam.h" > > Spurious line removal. Removed. > > 3. > A password needs to be provided too, if the sender demands > password > authentication. It can be provided in the > <varname>primary_conninfo</varname> string, or in a separate > - <filename>~/.pgpass</filename> file on the standby server (use > - <literal>replication</literal> as the database name). > - Do not specify a database name in the > - <varname>primary_conninfo</varname> string. > + <filename>~/.pgpass</filename> file on the standby server. > + </para> > + <para> > + Specify <literal>dbname</literal> in > + <varname>primary_conninfo</varname> string to allow > synchronization > + of slots from the primary server to the standby server. > + This will only be used for slot synchronization. It is ignored > + for streaming. > > Is there a reason to remove part of the earlier sentence "use > <literal>replication</literal> as the database name"? Added it back. > > 4. > + <primary><varname>enable_syncslot</varname> configuration > parameter</primary> > + </indexterm> > + </term> > + <listitem> > + <para> > + It enables a physical standby to synchronize logical failover slots > + from the primary server so that logical subscribers are not blocked > + after failover. > + </para> > + <para> > + It is enabled by default. This parameter can only be set in the > + <filename>postgresql.conf</filename> file or on the server > command line. > + </para> > > I think you forgot to update the documentation for the default value of this > variable. Updated. > > 5. > + * a) start the logical replication workers for every enabled subscription > + * when not in standby_mode > + * b) start the slot-sync worker for logical failover slots synchronization > + * from the primary server when in standby_mode. > > Either use a full stop after both lines or none of these. Added a full stop. > > 6. > +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); > > There shouldn't be space between * and the worker. Removed, and added the type to typedefs.list. > > 7. > + if (!SlotSyncWorker->hdr.in_use) > + { > + LWLockRelease(SlotSyncWorkerLock); > + ereport(ERROR, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg("replication slot-sync worker not initialized, " > + "cannot attach"))); > + } > + > + if (SlotSyncWorker->hdr.proc) > + { > + LWLockRelease(SlotSyncWorkerLock); > + ereport(ERROR, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg("replication slot-sync worker is " > + "already running, cannot attach"))); > + } > > Using slot-sync in the error messages looks a bit odd to me. Can we use > "replication slot sync worker ..." in both these and other similar messages? I > think it would be better if we don't split the messages into multiple lines in > these cases as messages don't appear too long to me. Changed as suggested. > > 8. > +/* > + * Detach the worker from DSM and update 'proc' and 'in_use'. > + * Logical replication launcher will come to know using these > + * that the worker has shutdown. > + */ > +void > +slotsync_worker_detach(int code, Datum arg) { > > I think the reference to DSM is leftover from the previous version of the patch. > Can we change the above comments as per the new code? Changed. > > 9. > +static bool > +slotsync_worker_launch() > { > ... > + /* TODO: do we really need 'generation', analyse more here */ > + worker->hdr.generation++; > > We should do something about this TODO. As per my understanding, we don't > need a generation number for the slot sync worker as we have one such worker > but I guess the patch requires it because we are using existing logical > replication worker infrastructure. This brings the question of whether we really > need a separate SlotSyncWorkerInfo or if we can use existing > LogicalRepWorker and distinguish it with LogicalRepWorkerType? I guess you > didn't use it because most of the fields in LogicalRepWorker will be unused for > slot sync worker. Will think about this one and update in next version. > > 10. > + * Can't use existing functions like 'get_database_oid' from > +dbcommands.c for > + * validity purpose as they need db connection. > + */ > +static bool > +validate_dbname(const char *dbname) > > I don't know how important it is to validate the dbname before launching the > sync slot worker because anyway after launching, it will give an error while > initializing the connection if the dbname is invalid. But, if we think it is really > required, did you consider using GetDatabaseTuple()? Yes, we could export GetDatabaseTuple. Apart from this, I am thinking is it possible to release the restriction for the dbname. For example, slot sync worker could always connect to the 'template1' as the worker doesn't update the database objects. Although I didn't find some examples on server side, but some client commands(e.g. pg_upgrade) will connect to template1 to check some global objects. (Just FYI, the previous version patch used a replication command which may avoid the dbname but was replaced with SELECT to improve the flexibility and avoid introducing new command.) Best Regards, Hou zj ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-21 05:45 shveta malik <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 0 siblings, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-21 05:45 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Tue, Nov 21, 2023 at 10:01 AM Zhijie Hou (Fujitsu) <[email protected]> wrote: > > On Saturday, November 18, 2023 6:46 PM Amit Kapila <[email protected]> wrote: > > > > On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > > > On 11/17/23 2:46 AM, Zhijie Hou (Fujitsu) wrote: > > > > On Tuesday, November 14, 2023 10:27 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > > > > > I feel the WaitForWALToBecomeAvailable may not be the best place to > > > > shutdown slotsync worker and drop slots. There could be other > > > > reasons(other than > > > > promotion) as mentioned in comments in case XLOG_FROM_STREAM to > > > > reach the code there. I thought if the intention is to stop slotsync > > > > workers on promotion, maybe FinishWalRecovery() is a better place to > > > > do it as it's indicating the end of recovery and XLogShutdownWalRcv is also > > called in it. > > > > > > I can see that slotsync_drop_initiated_slots() has been moved in > > > FinishWalRecovery() in v35. That looks ok. > > > > > > > > > > More Review for v35-0002* > > Thanks for the comments. > > > ============================ > > 1. > > + ereport(WARNING, > > + errmsg("skipping slots synchronization as primary_slot_name " > > + "is not set.")); > > > > There is no need to use a full stop at the end for WARNING messages and as > > previously mentioned, let's not split message lines in such cases. There are > > other messages in the patch with similar problems, please fix those as well. > > Adjusted. > > > > > 2. > > +slotsync_checks() > > { > > ... > > ... > > + /* The hot_standby_feedback must be ON for slot-sync to work */ if > > + (!hot_standby_feedback) { ereport(WARNING, errmsg("skipping slots > > + synchronization as hot_standby_feedback " > > + "is off.")); > > > > This message has the same problem as mentioned in the previous comment. > > Additionally, I think either atop slotsync_checks or along with GUC check we > > should write comments as to why we expect these values to be set for slot sync > > to work. > > Added comments for these cases. > > > > > 3. > > + /* The worker is running already */ > > + if (SlotSyncWorker &&SlotSyncWorker->hdr.in_use && > > + SlotSyncWorker->hdr.proc) > > > > The spacing for both the &&'s has problems. You need a space after the first > > && and the second && should be in the prior line. > > Adjusted. > > > > > 4. > > + LauncherRereadConfig(&recheck_slotsync); > > + > > } > > > > An empty line after LauncherRereadConfig() is not required. > > > > 5. > > +static void > > +LauncherRereadConfig(bool *ss_recheck) > > +{ > > + char *conninfo = pstrdup(PrimaryConnInfo); > > + char *slotname = pstrdup(PrimarySlotName); > > + bool syncslot = enable_syncslot; > > + bool feedback = hot_standby_feedback; > > > > Can we change the variable name 'feedback' to 'standbyfeedback' to make it > > slightly more descriptive? > > Changed. > > > > > 6. The logic to recheck the slot_sync related parameters in > > LauncherMain() is not very clear. IIUC, if after reload config any parameter is > > changed, we just seem to be checking the validity of the changed parameter > > but not restarting the slot sync worker, is that correct? If so, what if dbname is > > changed, don't we need to restart the slot-sync worker and re-initialize the > > connection; similarly slotname change also needs some thoughts. Also, if all the > > parameters are valid we seem to be re-launching the slot-sync worker without > > first stopping it which doesn't seem correct, am I missing something in this > > logic? > > I think the slot sync worker will be stopped in LauncherRereadConfig() if GUC changed > and new slot sync worker will be started in next loop in LauncherMain(). yes, LauncherRereadConfig will stop the worker on any parameter change and will set recheck_slotsync(). On finding this flag as true, LauncherMain will redo all the validations and restart slot-sync worker if needed. Yes, we do need to stop and relaunch slotsync workers on dbname change as well. This is currently missing inLauncherRereadConfig (). Regarding slot name change,we are already doing it, we are already checking PrimarySlotName in LauncherRereadConfig() > > > > 7. > > @@ -524,6 +525,25 @@ CreateDecodingContext(XLogRecPtr start_lsn, > > errmsg("replication slot \"%s\" was not created in this database", > > NameStr(slot->data.name)))); > > > > + in_recovery = RecoveryInProgress(); > > + > > + /* > > + * Do not allow consumption of a "synchronized" slot until the standby > > + * gets promoted. Also do not allow consumption of slots with > > + sync_state > > + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be > > + * used. > > + */ > > + if ((in_recovery && (slot->data.sync_state != SYNCSLOT_STATE_NONE)) || > > + slot->data.sync_state == SYNCSLOT_STATE_INITIATED) > > + ereport(ERROR, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg("cannot use replication slot \"%s\" for logical decoding", > > + NameStr(slot->data.name)), in_recovery ? > > + errdetail("This slot is being synced from the primary server.") : > > + errdetail("This slot was not synced completely from the primary > > + server."), errhint("Specify another replication slot."))); > > + > > > > If we are planning to drop slots in state SYNCSLOT_STATE_INITIATED at the > > time of promotion, don't we need to just have an assert or elog(ERROR, .. for > > non-recovery cases as such cases won't be reachable? If so, I think we can > > separate out that case here. > > Adjusted the codes as suggested. > > > > > 8. > > wait_for_primary_slot_catchup() > > { > > ... > > + /* Check if this standby is promoted while we are waiting */ if > > + (!RecoveryInProgress()) { > > + /* > > + * The remote slot didn't pass the locally reserved position at > > + * the time of local promotion, so it's not safe to use. > > + */ > > + ereport( > > + WARNING, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg( > > + "slot-sync wait for slot %s interrupted by promotion, " > > + "slot creation aborted", remote_slot->name))); pfree(cmd.data); return > > + false; } > > ... > > } > > > > Shouldn't this be an Assert because a slot-sync worker shouldn't exist for > > non-standby servers? > > Changed to Assert. > > > > > 9. > > wait_for_primary_slot_catchup() > > { > > ... > > + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); > > + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) { > > + ereport(WARNING, (errmsg("slot \"%s\" disappeared from the primary > > + server," > > + " slot creation aborted", remote_slot->name))); pfree(cmd.data); > > + walrcv_clear_result(res); return false; > > > > If the slot on primary disappears, shouldn't this part of the code somehow > > ensure to remove the slot on standby as well? If it is taken at some other point > > in time then at least we should write a comment here to state how it is taken > > care of. I think this comment also applies to a few other checks following this > > check. > > I adjusted the code here to not persist the slots if the slot disappeared or invalidated > on primary, so that the local slot will get dropped when releasing. > > > > > 10. > > + /* > > + * It is possible to get null values for lsns and xmin if slot is > > + * invalidated on the primary server, so handle accordingly. > > + */ > > + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); > > > > We can say LSN and Xmin in the above comment to make it easier to > > read/understand. > > Changed. > > > > > 11. > > /* > > + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin > > + * are expected to be valid/non-null, so assert if found null. > > + */ > > > > No need to explicitly say about assert, it is clear from the code. We can slightly > > change this comment to: "Once we got valid restart_lsn, then confirmed_lsn > > and catalog_xmin are expected to be valid/non-null." > > Changed. > > > > > 12. > > + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || > > + TransactionIdPrecedes(remote_slot->catalog_xmin, > > + MyReplicationSlot->data.catalog_xmin)) > > + { > > + if (!wait_for_primary_slot_catchup(wrconn, remote_slot)) { > > + /* > > + * The remote slot didn't catch up to locally reserved position. > > + * But still persist it and attempt the wait and sync in next > > + * sync-cycle. > > + */ > > + if (MyReplicationSlot->data.persistency != RS_PERSISTENT) { > > + ReplicationSlotPersist(); *slot_updated = true; } > > > > I think the reason to persist in this case is because next time local restart_lsn can > > be ahead than the current location and it can take more time to create such a > > slot. We can probably mention the same in the comments. > > Updated the comments. > > Here is the V37 patch set which addressed comments above and [1]. > > [1] https://www.postgresql.org/message-id/CAA4eK1%2BP9R3GO2rwGBg2EOh%3DuYjWUSEOHD8yvs4Je8WYa2RHag%40mail... > > Best Regards, > Hou zj ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-21 06:01 shveta malik <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-21 06:01 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Tue, Nov 21, 2023 at 10:02 AM Zhijie Hou (Fujitsu) <[email protected]> wrote: > > On Friday, November 17, 2023 7:39 PM Amit Kapila <[email protected]> wrote: > > > > On Thu, Nov 16, 2023 at 5:34 PM shveta malik <[email protected]> > > wrote: > > > > > > PFA v35. > > > > > > > Review v35-0002* > > ============== > > Thanks for the comments. > > > 1. > > As quoted in the commit message, > > > > > If a logical slot is invalidated on the primary, slot on the standby is also > > invalidated. If a logical slot on the primary is valid but is invalidated on the > > standby due to conflict (say required rows removed on the primary), then that > > slot is dropped and recreated on the standby in next sync-cycle. > > It is okay to recreate such slots as long as these are not consumable on the > > standby (which is the case currently). > > > > > > > I think this won't happen normally because of the physical slot and > > hot_standby_feedback but probably can occur in cases like if the user > > temporarily switches hot_standby_feedback from on to off. Are there any other > > reasons? I think we can mention the cases along with it as well at least for now. > > Additionally, I think this should be covered in code comments as well. > > I will collect all these cases and update in next version. > > > > > 2. > > #include "postgres.h" > > - > > +#include "access/genam.h" > > > > Spurious line removal. > > Removed. > > > > > 3. > > A password needs to be provided too, if the sender demands > > password > > authentication. It can be provided in the > > <varname>primary_conninfo</varname> string, or in a separate > > - <filename>~/.pgpass</filename> file on the standby server (use > > - <literal>replication</literal> as the database name). > > - Do not specify a database name in the > > - <varname>primary_conninfo</varname> string. > > + <filename>~/.pgpass</filename> file on the standby server. > > + </para> > > + <para> > > + Specify <literal>dbname</literal> in > > + <varname>primary_conninfo</varname> string to allow > > synchronization > > + of slots from the primary server to the standby server. > > + This will only be used for slot synchronization. It is ignored > > + for streaming. > > > > Is there a reason to remove part of the earlier sentence "use > > <literal>replication</literal> as the database name"? > > Added it back. > > > > > 4. > > + <primary><varname>enable_syncslot</varname> configuration > > parameter</primary> > > + </indexterm> > > + </term> > > + <listitem> > > + <para> > > + It enables a physical standby to synchronize logical failover slots > > + from the primary server so that logical subscribers are not blocked > > + after failover. > > + </para> > > + <para> > > + It is enabled by default. This parameter can only be set in the > > + <filename>postgresql.conf</filename> file or on the server > > command line. > > + </para> > > > > I think you forgot to update the documentation for the default value of this > > variable. > > Updated. > > > > > 5. > > + * a) start the logical replication workers for every enabled subscription > > + * when not in standby_mode > > + * b) start the slot-sync worker for logical failover slots synchronization > > + * from the primary server when in standby_mode. > > > > Either use a full stop after both lines or none of these. > > Added a full stop. > > > > > 6. > > +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); > > > > There shouldn't be space between * and the worker. > > Removed, and added the type to typedefs.list. > > > > > 7. > > + if (!SlotSyncWorker->hdr.in_use) > > + { > > + LWLockRelease(SlotSyncWorkerLock); > > + ereport(ERROR, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg("replication slot-sync worker not initialized, " > > + "cannot attach"))); > > + } > > + > > + if (SlotSyncWorker->hdr.proc) > > + { > > + LWLockRelease(SlotSyncWorkerLock); > > + ereport(ERROR, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg("replication slot-sync worker is " > > + "already running, cannot attach"))); > > + } > > > > Using slot-sync in the error messages looks a bit odd to me. Can we use > > "replication slot sync worker ..." in both these and other similar messages? I > > think it would be better if we don't split the messages into multiple lines in > > these cases as messages don't appear too long to me. > > Changed as suggested. > > > > > 8. > > +/* > > + * Detach the worker from DSM and update 'proc' and 'in_use'. > > + * Logical replication launcher will come to know using these > > + * that the worker has shutdown. > > + */ > > +void > > +slotsync_worker_detach(int code, Datum arg) { > > > > I think the reference to DSM is leftover from the previous version of the patch. > > Can we change the above comments as per the new code? > > Changed. > > > > > 9. > > +static bool > > +slotsync_worker_launch() > > { > > ... > > + /* TODO: do we really need 'generation', analyse more here */ > > + worker->hdr.generation++; > > > > We should do something about this TODO. As per my understanding, we don't > > need a generation number for the slot sync worker as we have one such worker > > but I guess the patch requires it because we are using existing logical > > replication worker infrastructure. This brings the question of whether we really > > need a separate SlotSyncWorkerInfo or if we can use existing > > LogicalRepWorker and distinguish it with LogicalRepWorkerType? I guess you > > didn't use it because most of the fields in LogicalRepWorker will be unused for > > slot sync worker. > > Will think about this one and update in next version. > > > > > 10. > > + * Can't use existing functions like 'get_database_oid' from > > +dbcommands.c for > > + * validity purpose as they need db connection. > > + */ > > +static bool > > +validate_dbname(const char *dbname) > > > > I don't know how important it is to validate the dbname before launching the > > sync slot worker because anyway after launching, it will give an error while > > initializing the connection if the dbname is invalid. But, if we think it is really > > required, did you consider using GetDatabaseTuple()? > > Yes, we could export GetDatabaseTuple. Apart from this, I am thinking is it possible to > release the restriction for the dbname. For example, slot sync worker could > always connect to the 'template1' as the worker doesn't update the > database objects. Although I didn't find some examples on server side, but some > client commands(e.g. pg_upgrade) will connect to template1 to check some global > objects. We use this dbname for 2 purposes: a) which you pointed out i.e. to have db connection in sync worker, b) to make connection to primary server's db so that we can run SELECT queries there. Thought of adding this point, so that we have complete info before deciding the next step. (Just FYI, the previous version patch used a replication command which > may avoid the dbname but was replaced with SELECT to improve the flexibility and > avoid introducing new command.) Would like to add more info here. We had a LIST command in launcher.c for multi-worker design case to fetch dbids from primary. But for a single worker case we do not need that info, so we got rid of that command. In slotsync.c we always had usage of 'SELECT' query and never had any cmd implemented. So we never replaced replication-cmd with 'SELECT' in any of the patches. Why we had cmd usage in launcher.c is due the fact that launcher originally did not have any db-connection and we did not want to change that . And since running 'SELECT' query through exposed libpq APIs need db-connection, thus we decided to retain replication-cmd in the launcher. While in slotsync.c we had to run multiple queries to get different information, and thus to retain the flexibility and ease of extension over replication-cmds, we decided to go with SELECT and thus opted for db-connection as needed by libpq APIs. thanks Shveta ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-21 08:26 shveta malik <[email protected]> parent: Zhijie Hou (Fujitsu) <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-21 08:26 UTC (permalink / raw) To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; Drouvot, Bertrand <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Tue, Nov 21, 2023 at 10:02 AM Zhijie Hou (Fujitsu) <[email protected]> wrote: > > On Friday, November 17, 2023 7:39 PM Amit Kapila <[email protected]> wrote: > > > > On Thu, Nov 16, 2023 at 5:34 PM shveta malik <[email protected]> > > wrote: > > > > > > PFA v35. > > > > > > > Review v35-0002* > > ============== > > Thanks for the comments. > > > 1. > > As quoted in the commit message, > > > > > If a logical slot is invalidated on the primary, slot on the standby is also > > invalidated. If a logical slot on the primary is valid but is invalidated on the > > standby due to conflict (say required rows removed on the primary), then that > > slot is dropped and recreated on the standby in next sync-cycle. > > It is okay to recreate such slots as long as these are not consumable on the > > standby (which is the case currently). > > > > > > > I think this won't happen normally because of the physical slot and > > hot_standby_feedback but probably can occur in cases like if the user > > temporarily switches hot_standby_feedback from on to off. Are there any other > > reasons? I think we can mention the cases along with it as well at least for now. > > Additionally, I think this should be covered in code comments as well. > > I will collect all these cases and update in next version. > > > > > 2. > > #include "postgres.h" > > - > > +#include "access/genam.h" > > > > Spurious line removal. > > Removed. > > > > > 3. > > A password needs to be provided too, if the sender demands > > password > > authentication. It can be provided in the > > <varname>primary_conninfo</varname> string, or in a separate > > - <filename>~/.pgpass</filename> file on the standby server (use > > - <literal>replication</literal> as the database name). > > - Do not specify a database name in the > > - <varname>primary_conninfo</varname> string. > > + <filename>~/.pgpass</filename> file on the standby server. > > + </para> > > + <para> > > + Specify <literal>dbname</literal> in > > + <varname>primary_conninfo</varname> string to allow > > synchronization > > + of slots from the primary server to the standby server. > > + This will only be used for slot synchronization. It is ignored > > + for streaming. > > > > Is there a reason to remove part of the earlier sentence "use > > <literal>replication</literal> as the database name"? > > Added it back. > > > > > 4. > > + <primary><varname>enable_syncslot</varname> configuration > > parameter</primary> > > + </indexterm> > > + </term> > > + <listitem> > > + <para> > > + It enables a physical standby to synchronize logical failover slots > > + from the primary server so that logical subscribers are not blocked > > + after failover. > > + </para> > > + <para> > > + It is enabled by default. This parameter can only be set in the > > + <filename>postgresql.conf</filename> file or on the server > > command line. > > + </para> > > > > I think you forgot to update the documentation for the default value of this > > variable. > > Updated. > > > > > 5. > > + * a) start the logical replication workers for every enabled subscription > > + * when not in standby_mode > > + * b) start the slot-sync worker for logical failover slots synchronization > > + * from the primary server when in standby_mode. > > > > Either use a full stop after both lines or none of these. > > Added a full stop. > > > > > 6. > > +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); > > > > There shouldn't be space between * and the worker. > > Removed, and added the type to typedefs.list. > > > > > 7. > > + if (!SlotSyncWorker->hdr.in_use) > > + { > > + LWLockRelease(SlotSyncWorkerLock); > > + ereport(ERROR, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg("replication slot-sync worker not initialized, " > > + "cannot attach"))); > > + } > > + > > + if (SlotSyncWorker->hdr.proc) > > + { > > + LWLockRelease(SlotSyncWorkerLock); > > + ereport(ERROR, > > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > > + errmsg("replication slot-sync worker is " > > + "already running, cannot attach"))); > > + } > > > > Using slot-sync in the error messages looks a bit odd to me. Can we use > > "replication slot sync worker ..." in both these and other similar messages? I > > think it would be better if we don't split the messages into multiple lines in > > these cases as messages don't appear too long to me. > > Changed as suggested. > > > > > 8. > > +/* > > + * Detach the worker from DSM and update 'proc' and 'in_use'. > > + * Logical replication launcher will come to know using these > > + * that the worker has shutdown. > > + */ > > +void > > +slotsync_worker_detach(int code, Datum arg) { > > > > I think the reference to DSM is leftover from the previous version of the patch. > > Can we change the above comments as per the new code? > > Changed. > > > > > 9. > > +static bool > > +slotsync_worker_launch() > > { > > ... > > + /* TODO: do we really need 'generation', analyse more here */ > > + worker->hdr.generation++; > > > > We should do something about this TODO. As per my understanding, we don't > > need a generation number for the slot sync worker as we have one such worker > > but I guess the patch requires it because we are using existing logical > > replication worker infrastructure. Yes, we do not need generation, but since we want to use existing logical-rep worker infrastructure, we can retain generation but can keep it as zero always for the slot-sync worker case. > > This brings the question of whether we really > > need a separate SlotSyncWorkerInfo or if we can use existing > > LogicalRepWorker and distinguish it with LogicalRepWorkerType? I guess you > > didn't use it because most of the fields in LogicalRepWorker will be unused for > > slot sync worker. Yes, right. If we use LogicalRepWorker in the slot-sync worker, then it will be a task to keep a check (even in future) that no-one should end up using uninitialized fields in slot-sync code. That is why shifting common fields to LogicalWorkerHeader and using that in SlotSyncWorkerInfo and LogicalRepWorker seems a better approach to me. > > Will think about this one and update in next version. > > > > > 10. > > + * Can't use existing functions like 'get_database_oid' from > > +dbcommands.c for > > + * validity purpose as they need db connection. > > + */ > > +static bool > > +validate_dbname(const char *dbname) > > > > I don't know how important it is to validate the dbname before launching the > > sync slot worker because anyway after launching, it will give an error while > > initializing the connection if the dbname is invalid. But, if we think it is really > > required, did you consider using GetDatabaseTuple()? > > Yes, we could export GetDatabaseTuple. Apart from this, I am thinking is it possible to > release the restriction for the dbname. For example, slot sync worker could > always connect to the 'template1' as the worker doesn't update the > database objects. Although I didn't find some examples on server side, but some > client commands(e.g. pg_upgrade) will connect to template1 to check some global > objects. (Just FYI, the previous version patch used a replication command which > may avoid the dbname but was replaced with SELECT to improve the flexibility and > avoid introducing new command.) > > Best Regards, > Hou zj > > ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-23 08:59 shveta malik <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-23 08:59 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Mon, Nov 20, 2023 at 4:28 PM Amit Kapila <[email protected]> wrote: > > On Sat, Nov 18, 2023 at 4:15 PM Amit Kapila <[email protected]> wrote: > > > > On Fri, Nov 17, 2023 at 5:18 PM Drouvot, Bertrand > > <[email protected]> wrote: > > > > More Review for v35-0002* > > ============================ > > > Thanks for the feedback. Please find the patch attached and my comments inline. > More review of v35-0002* > ==================== > 1. > +/* > + * Helper function to check if local_slot is present in remote_slots list. > + * > + * It also checks if logical slot is locally invalidated i.e. invalidated on > + * the standby but valid on the primary server. If found so, it sets > + * locally_invalidated to true. > + */ > +static bool > +slot_exists_in_list(ReplicationSlot *local_slot, List *remote_slots, > + bool *locally_invalidated) > > The name of the function is a bit misleading because it checks the > validity of the slot not only whether it exists in remote_list. Would > it be better to name it as ValidateSyncSlot() or something along those > lines? > Sure, updated the name. > 2. > +static long > +synchronize_slots(WalReceiverConn *wrconn) > { > ... > + /* Construct query to get slots info from the primary server */ > + initStringInfo(&s); > + construct_slot_query(&s); > ... > + if (remote_slot->conflicting) > + remote_slot->invalidated = get_remote_invalidation_cause(wrconn, > + remote_slot->name); > ... > > +static ReplicationSlotInvalidationCause > +get_remote_invalidation_cause(WalReceiverConn *wrconn, char *slot_name) > { > ... > + appendStringInfo(&cmd, > + "SELECT pg_get_slot_invalidation_cause(%s)", > + quote_literal_cstr(slot_name)); > + res = walrcv_exec(wrconn, cmd.data, 1, slotRow); > > Do we really need to query a second time to get the invalidation > cause? Can we adjust the slot_query to get it in one round trip? I > think this may not optimize much because the patch uses second round > trip only for invalidated slots but still looks odd. So unless the > query becomes too complicated, we should try to achive it one round > trip. > Modified the query to fetch all the info at once. > 3. > +static long > +synchronize_slots(WalReceiverConn *wrconn) > +{ > ... > ... > + /* The syscache access needs a transaction env. */ > + StartTransactionCommand(); > + > + /* Make things live outside TX context */ > + MemoryContextSwitchTo(oldctx); > + > + /* Construct query to get slots info from the primary server */ > + initStringInfo(&s); > + construct_slot_query(&s); > + > + elog(DEBUG2, "slot-sync worker's query:%s \n", s.data); > + > + /* Execute the query */ > + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); > > It is okay to perform the above query execution outside the > transaction context but I would like to know the reason for the same. > Do we want to retain anything beyond the transaction context or is > there some other reason to do this outside the transaction context? > Modified the comment with the reason. We need to start a transaction for syscache access. We can end it as soon as walrcv_exec() is over, but we need the tuple-results to be accessed even after that, thus those should not be allocated in TopTransactionContext. > 4. > +static void > +construct_slot_query(StringInfo s) > +{ > + /* > + * Fetch data for logical failover slots with sync_state either as > + * SYNCSLOT_STATE_NONE or SYNCSLOT_STATE_READY. > + */ > + appendStringInfo(s, > + "SELECT slot_name, plugin, confirmed_flush_lsn," > + " restart_lsn, catalog_xmin, two_phase, conflicting, " > + " database FROM pg_catalog.pg_replication_slots" > + " WHERE failover and sync_state != 'i'"); > +} > > Why would the sync_state on the primary server be any valid value? I > thought it was set only on physical standby. I think it is better to > mention the reason for using the sync state and or failover flag in > the above comments. The current comment doesn't seem of much use as it > just states what is evident from the query. Updated the reason in comment. It is mainly for cascading standby to fetch correct slots. > > 5. > * This check should never pass as on the primary server, we have waited > + * for the standby's confirmation before updating the logical slot. But to > + * take care of any bug in that flow, we should retain this check. > + */ > + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) > + { > + elog(LOG, "skipping sync of slot \"%s\" as the received slot-sync " > + "LSN %X/%X is ahead of the standby position %X/%X", > + remote_slot->name, > + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), > + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); > + > > This should be elog(ERROR, ..). Normally, we use elog(ERROR, ...) for > such unexpected cases. And, you don't need to explicitly mention the > last sentence in the comment: "But to take care of any bug in that > flow, we should retain this check.". > Sure, modified. > 6. > +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, > + bool *slot_updated) > { > ... > + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) > + { > + ereport(WARNING, > + errmsg("not synchronizing slot %s; synchronization would" > + " move it backwards", remote_slot->name)); > > I think here elevel should be LOG because user can't do much about > this. Do we use ';' at other places in the message? But when can we > hit this case? We can add some comments to state in which scenario > this possible. OTOH, if this is sort of can't happen case and we have > kept it to avoid any sort of inconsistency then we can probably use > elog(ERROR, .. with approapriate LSN locations, so that later the > problem could be debugged. > Converted to ERROR and updated comment > 7. > +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, > + bool *slot_updated) > { > ... > + > + StartTransactionCommand(); > + > + /* Make things live outside TX context */ > + MemoryContextSwitchTo(oldctx); > + > ... > > Similar to one of the previous comments, it is not clear to me why the > patch is doing a memory context switch here. Can we add a comment? > I have removed the memory-context-switch here as the results are all consumed within the span of transaction, so we do not need to retain those even after commit of txn for this particular case. > 8. > + /* User created slot with the same name exists, raise ERROR. */ > + else if (sync_state == SYNCSLOT_STATE_NONE) > + { > + ereport(ERROR, > + errmsg("not synchronizing slot %s; it is a user created slot", > + remote_slot->name)); > + } > > Won't we need error_code in this error? Also, the message doesn't seem > to follow the code's usual style. Modified. I have added errdetail as well, but not sure what we can add as error-hint, Shall we add something like: Try renaming existing slot. > > 9. > +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, > + bool *slot_updated) > { > ... > + else > + { > + TransactionId xmin_horizon = InvalidTransactionId; > + ReplicationSlot *slot; > + > + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, > + remote_slot->two_phase, false); > + slot = MyReplicationSlot; > + > + SpinLockAcquire(&slot->mutex); > + slot->data.database = get_database_oid(remote_slot->database, false); > + > + /* Mark it as sync initiated by slot-sync worker */ > + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; > + slot->data.failover = true; > + > + namestrcpy(&slot->data.plugin, remote_slot->plugin); > + SpinLockRelease(&slot->mutex); > + > + ReplicationSlotReserveWal(); > + > > How and when will this init state (SYNCSLOT_STATE_INITIATED) persist to disk? This will be inside wait_for_primary_and_sync. I have reorganized code here (removed wait_for_primary_and_sync) to make it more readable. > > 10. > + if (slot_updated) > + SlotSyncWorker->last_update_time = now; > + > + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, > + now, WORKER_INACTIVITY_THRESHOLD_MS)) > > Empty line between if/else if is not required. > This is added by pg_indent. Not sure how we can correct it. > 11. > +static WalReceiverConn * > +remote_connect() > +{ > + WalReceiverConn *wrconn = NULL; > + char *err; > + > + wrconn = walrcv_connect(PrimaryConnInfo, true, false, "slot-sync", &err); > + if (wrconn == NULL) > + ereport(ERROR, > + (errmsg("could not connect to the primary server: %s", err))); > > Let's use appname similar to what we do for "walreceiver" as shown below: > /* Establish the connection to the primary for XLOG streaming */ > wrconn = walrcv_connect(conninfo, false, false, > cluster_name[0] ? cluster_name : "walreceiver", > &err); > if (!wrconn) > ereport(ERROR, > (errcode(ERRCODE_CONNECTION_FAILURE), > errmsg("could not connect to the primary server: %s", err))); > > Some proposals for default appname "slotsynchronizer", "slotsync > worker". Also, use the same error code as used by "walreceiver". Modified. > > 12. Do we need the handling of the slotsync worker in > GetBackendTypeDesc()? Please check without that what value this patch > displays for backend_type. It currently displays "slot sync worker'. It is the same desc which launcher has launched this worker with (snprintf(bgw.bgw_type, BGW_MAXLEN, "slot sync worker")). postgres=# select backend_type from pg_stat_activity; backend_type ------------------------------ logical replication launcher slot sync worker ....... For slot sync and logical launcher, BackendType is B_BG_WORKER and thus pg_stat_get_activity() for this type displays backend_type as the one given during background process registration and thus we get these correctly. But pg_stat_get_io() does not have the same implementation, it displays 'background worker' as the description. I think slot-sync and logical launcher are one of these entries postgres=# select backend_type from pg_stat_io; backend_type --------------------- autovacuum launcher .. background worker background worker background worker background worker background worker background writer ..... > > 13. > +/* > + * Re-read the config file. > + * > + * If primary_conninfo has changed, reconnect to primary. > + */ > +static void > +slotsync_reread_config(WalReceiverConn **wrconn) > +{ > + char *conninfo = pstrdup(PrimaryConnInfo); > + > + ConfigReloadPending = false; > + ProcessConfigFile(PGC_SIGHUP); > + > + /* Reconnect if GUC primary_conninfo got changed */ > + if (strcmp(conninfo, PrimaryConnInfo) != 0) > + { > + if (*wrconn) > + walrcv_disconnect(*wrconn); > + > + *wrconn = remote_connect(); > > I think we should exit the worker in this case and allow it to > reconnect. See the similar handling in maybe_reread_subscription(). > One effect of not doing is that the dbname patch has used in > ReplSlotSyncWorkerMain() will become inconsistent. > Modified as suggested. > 14. > +void > +ReplSlotSyncWorkerMain(Datum main_arg) > +{ > ... > ... > + /* > + * If the standby has been promoted, skip the slot synchronization process. > + * > + * Although the startup process stops all the slot-sync workers on > + * promotion, the launcher may not have realized the promotion and could > + * start additional workers after that. Therefore, this check is still > + * necessary to prevent these additional workers from running. > + */ > + if (PromoteIsTriggered()) > + exit(0); > ... > ... > + /* Check if got promoted */ > + if (!RecoveryInProgress()) > + { > + /* > + * Drop the slots for which sync is initiated but not yet > + * completed i.e. they are still waiting for the primary server to > + * catch up. > + */ > + slotsync_drop_initiated_slots(); > + ereport(LOG, > + errmsg("exiting slot-sync woker on promotion of standby")); > > I think we should never reach this code in non-standby mode. It should > elog(ERROR,.. Can you please explain why promotion handling is > required here? I will handle this in the next version. It needs some more thoughts, especially on how 'PromoteIsTriggered' can be removed. > > 15. > @@ -190,6 +190,8 @@ static const char *const BuiltinTrancheNames[] = { > "LogicalRepLauncherDSA", > /* LWTRANCHE_LAUNCHER_HASH: */ > "LogicalRepLauncherHash", > + /* LWTRANCHE_SLOTSYNC_DSA: */ > + "SlotSyncWorkerDSA", > }; > ... > ... > + LWTRANCHE_SLOTSYNC_DSA, > LWTRANCHE_FIRST_USER_DEFINED, > } BuiltinTrancheIds; > > These are not used in the patch. > Removed. > 16. > +/* ------------------------------- > + * LIST_DBID_FOR_FAILOVER_SLOTS command > + * ------------------------------- > + */ > +typedef struct ListDBForFailoverSlotsCmd > +{ > + NodeTag type; > + List *slot_names; > +} ListDBForFailoverSlotsCmd; > > ... > > +/* > + * Failover logical slots data received from remote. > + */ > +typedef struct WalRcvFailoverSlotsData > +{ > + Oid dboid; > +} WalRcvFailoverSlotsData; > > These structures don't seem to be used in the current version of the patch. Removed. > > 17. > --- a/src/include/replication/slot.h > +++ b/src/include/replication/slot.h > @@ -15,7 +15,6 @@ > #include "storage/lwlock.h" > #include "storage/shmem.h" > #include "storage/spin.h" > -#include "replication/walreceiver.h" > ... > ... > -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); > extern List *GetStandbySlotList(bool copy); > > Why the above two are removed as part of this patch? WaitForStandbyLSN() is no longer there, so that is why it was removed. I think it should have been removed from patch0001. WIll make this change in the next version where we have pacth0001 changes coming. Regarding header inclusion and 'ReplicationSlotDropAtPubNode' removal, not sure when those were removed. But my best guess is that the header inclusion chain has changed a little bit in patch. The tablesync.c uses ReplicationSlotDropAtPubNode which is part of subscriptioncmds.h. Now in our patch since tablesync.c includes subscriptioncmds.h and thus slot.h need not to extern it for tablesync.c. And if we can get rid of ReplicationSlotDropAtPubNode in slot.h, then walreceiver.h inclusion can also be removed as that was needed for 'WalReceiverConn' argument of ReplicationSlotDropAtPubNode. There could be other 'header inclusions' involved as well but this seems the primary reason. > -- > With Regards, > Amit Kapila. Attachments: [application/octet-stream] v38-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch (8.0K, ../../CAJpy0uD6dWUvBgy8MGdugf_Am4pLXTL_vqcwSeHO13v+Mzc9KA@mail.gmail.com/2-v38-0003-Allow-slot-sync-worker-to-wait-for-the-cascading.patch) download | inline diff: From 87d2bfb3c0bf9073ebeb2bfde6bdf514f6419cef Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Thu, 23 Nov 2023 12:49:05 +0530 Subject: [PATCH v38 3/3] Allow slot-sync worker to wait for the cascading standbys. The GUC standby_slot_names is needed to be set on first standby in order to allow it to wait for confirmation for cascading standbys before updating logical 'synced' slots in slot-sync worker. The intent is that the logical slots (synced ones) should not go ahead of cascading standbys. For the user created slots on first standby, we already have this wait logic in place in logical walsender and in pg_logical_slot_get_changes_guts(), but for synced slots (which can not be consumed yet), we need to make sure that they are not going ahead of cascading standbys and that is acheived by introducing the wait in slot-sync worker before we actually update the slots. --- src/backend/replication/logical/slotsync.c | 86 ++++++++++++++++++++-- src/backend/replication/walsender.c | 11 +-- src/include/replication/walsender.h | 5 ++ 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 9aa3562500..1aac8ce648 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -83,6 +83,9 @@ typedef struct RemoteSlot */ #define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 +static void ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots); + /* * Wait for remote slot to pass locally reserved position. * @@ -452,6 +455,52 @@ construct_slot_query(StringInfo s) " WHERE failover and sync_state != 'i'"); } +/* + * Wait for cascading physical standbys corresponding to physical slots + * specified in standby_slot_names GUC to confirm receiving given lsn. + */ +static void +wait_for_standby_confirmation(XLogRecPtr wait_for_lsn, + WalReceiverConn *wrconn) +{ + List *standby_slots; + + /* Nothing to be done */ + if (strcmp(standby_slot_names, "") == 0) + return; + + standby_slots = GetStandbySlotList(true); + + for (;;) + { + int rc; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + /* + * This will reload configuration and will refresh the standby_slots + * as well provided standby_slot_names GUC is changed by the user. + */ + ProcessSlotSyncInterrupts(&wrconn, &standby_slots); + + /* + * XXX: Is waiting for 5 second before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 5000L, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } +} + /* * Synchronize single slot to given position. * @@ -723,6 +772,7 @@ synchronize_slots(WalReceiverConn *wrconn) List *remote_slot_list = NIL; MemoryContext oldctx = CurrentMemoryContext; long naptime = WORKER_DEFAULT_NAPTIME_MS; + XLogRecPtr max_confirmed_lsn = 0; ListCell *cell; bool slot_updated = false; TimestampTz now; @@ -775,6 +825,9 @@ synchronize_slots(WalReceiverConn *wrconn) remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); Assert(!isnull); + if (remote_slot->confirmed_lsn > max_confirmed_lsn) + max_confirmed_lsn = remote_slot->confirmed_lsn; + /* * It is possible to get null values for LSN and Xmin if slot is * invalidated on the primary server, so handle accordingly. @@ -818,6 +871,17 @@ synchronize_slots(WalReceiverConn *wrconn) */ drop_obsolete_slots(remote_slot_list); + /* + * If there are cascading standbys, wait for their confirmation before we + * update synced logical slots locally. + * + * Instead of waiting on confirmation for lsn of each slot, let us wait + * once for confirmation on max_confirmed_lsn. If that is confirmed by + * each cascading standby, we are good to update all the slots. + */ + if (remote_slot_list) + wait_for_standby_confirmation(max_confirmed_lsn, wrconn); + /* Now sync the slots locally */ foreach(cell, remote_slot_list) { @@ -877,12 +941,18 @@ remote_connect(void) * If primary_conninfo has changed, reconnect to primary. */ static void -slotsync_reread_config(WalReceiverConn **wrconn) +slotsync_reread_config(WalReceiverConn **wrconn, List **standby_slots) { char *conninfo = pstrdup(PrimaryConnInfo); - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + /* + * Reload configs and recreate the standby_slot_names_list if GUC + * standby_slot_names changed. + */ + if (standby_slots) + WalSndRereadConfigAndReInitSlotList(standby_slots); + else + ProcessConfigFile(PGC_SIGHUP); /* Exit if GUC primary_conninfo got changed, let the launcher relaunch it */ if (strcmp(conninfo, PrimaryConnInfo) != 0) @@ -905,7 +975,8 @@ slotsync_reread_config(WalReceiverConn **wrconn) * Interrupt handler for main loop of slot sync worker. */ static void -ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn, + List **standby_slots) { CHECK_FOR_INTERRUPTS(); @@ -921,7 +992,10 @@ ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) if (ConfigReloadPending) - slotsync_reread_config(wrconn); + { + ConfigReloadPending = false; + slotsync_reread_config(wrconn, standby_slots); + } } /* @@ -982,7 +1056,7 @@ ReplSlotSyncWorkerMain(Datum main_arg) int rc; long naptime; - ProcessSlotSyncInterrupts(&wrconn); + ProcessSlotSyncInterrupts(&wrconn, NULL /* standby_slots */ ); /* Check if got promoted */ if (!RecoveryInProgress()) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 8e4b0c66be..3db1fa40ac 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1614,7 +1614,7 @@ PhysicalWakeupLogicalWalSnd(void) * Reload the config file and reinitialize the standby slot list if the GUC * standby_slot_names has changed. */ -static void +void WalSndRereadConfigAndReInitSlotList(List **standby_slots) { char *pre_standby_slot_names = pstrdup(standby_slot_names); @@ -1637,13 +1637,12 @@ WalSndRereadConfigAndReInitSlotList(List **standby_slots) * This function updates the passed standby_slots list, removing any slots that * have already caught up to or surpassed the given wait_for_lsn. */ -static void +void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) { ListCell *lc; - List *standby_slots_cpy = *standby_slots; - foreach(lc, standby_slots_cpy) + foreach(lc, *standby_slots) { char *name = lfirst(lc); XLogRecPtr restart_lsn = InvalidXLogRecPtr; @@ -1710,10 +1709,8 @@ WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) if (warningfmt) ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); - standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + *standby_slots = foreach_delete_current(*standby_slots, lc); } - - *standby_slots = standby_slots_cpy; } /* diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 1fcc22a127..c5c4714788 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -15,6 +15,7 @@ #include <signal.h> #include "access/xlogdefs.h" +#include "nodes/pg_list.h" /* * What to do with a snapshot in create replication slot command. @@ -50,7 +51,11 @@ extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, + List **standby_slots); extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); +extern List *WalSndGetStandbySlots(void); +extern void WalSndRereadConfigAndReInitSlotList(List **standby_slots); /* * Remember that we want to wakeup walsenders later -- 2.34.1 [application/octet-stream] v38-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch (129.6K, ../../CAJpy0uD6dWUvBgy8MGdugf_Am4pLXTL_vqcwSeHO13v+Mzc9KA@mail.gmail.com/3-v38-0001-Allow-logical-walsenders-to-wait-for-the-physica.patch) download | inline diff: From 6e33307777a4f611ab5d05118ee84623e74ae77c Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 21 Nov 2023 14:50:21 +0530 Subject: [PATCH v38 1/3] Allow logical walsenders to wait for the physical standbys A new property 'failover' is added at the slot level which is persistent information which specifies that this logical slot is enabled to be synced to the physical standbys so that logical replication can be resumed after failover. It is always false for physical slots. Users can set it during the create subscription or during pg_create_logical_replication_slot. Examples: create subscription mysub connection '..' publication mypub WITH (failover = true); --last arg SELECT * FROM pg_create_logical_replication_slot('myslot', 'pgoutput', false, true, true); Altering the failover option of the subscription is currently not permitted. However, this restriction may be lifted in future versions. This 'failover' is displayed as part of pg_replication_slots view. A new GUC standby_slot_names has been added. It is the list of physical replication slots that logical replication with failover enabled waits for. The intent of this wait is that no logical replication subscribers (with failover=true) should go ahead of physical replication standbys (corresponding to the physical slots in standby_slot_names). A new walreceiver API walrcv_alter_slot has been introduced to enable the failover of the slot on publisher node. --- contrib/test_decoding/expected/slot.out | 19 + contrib/test_decoding/sql/slot.sql | 6 + doc/src/sgml/catalogs.sgml | 12 + doc/src/sgml/config.sgml | 22 ++ doc/src/sgml/func.sgml | 11 +- doc/src/sgml/protocol.sgml | 51 +++ doc/src/sgml/ref/alter_subscription.sgml | 7 +- doc/src/sgml/ref/create_subscription.sgml | 24 ++ doc/src/sgml/system-views.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_functions.sql | 1 + src/backend/catalog/system_views.sql | 6 +- src/backend/commands/subscriptioncmds.c | 88 ++++- .../libpqwalreceiver/libpqwalreceiver.c | 44 ++- .../replication/logical/logicalfuncs.c | 13 + src/backend/replication/logical/tablesync.c | 57 ++- src/backend/replication/logical/worker.c | 40 +- src/backend/replication/repl_gram.y | 18 +- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/slot.c | 173 ++++++++- src/backend/replication/slotfuncs.c | 19 +- src/backend/replication/walreceiver.c | 2 +- src/backend/replication/walsender.c | 343 ++++++++++++++++-- .../utils/activity/wait_event_names.txt | 1 + src/backend/utils/misc/guc_tables.c | 14 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/pg_upgrade/info.c | 5 +- src/bin/pg_upgrade/pg_upgrade.c | 10 +- src/bin/pg_upgrade/pg_upgrade.h | 2 + src/bin/pg_upgrade/t/003_logical_slots.pl | 6 +- src/bin/psql/describe.c | 8 +- src/bin/psql/tab-complete.c | 3 +- src/include/catalog/pg_proc.dat | 14 +- src/include/catalog/pg_subscription.h | 7 + src/include/nodes/replnodes.h | 12 + src/include/replication/slot.h | 13 +- src/include/replication/walreceiver.h | 18 +- src/include/replication/walsender.h | 4 + src/include/replication/walsender_private.h | 7 + src/include/replication/worker_internal.h | 4 +- src/include/utils/guc_hooks.h | 3 + src/test/recovery/meson.build | 1 + src/test/recovery/t/006_logical_decoding.pl | 3 +- src/test/recovery/t/050_verify_slot_order.pl | 145 ++++++++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/subscription.out | 152 ++++---- src/tools/pgindent/typedefs.list | 2 + 47 files changed, 1248 insertions(+), 163 deletions(-) create mode 100644 src/test/recovery/t/050_verify_slot_order.pl diff --git a/contrib/test_decoding/expected/slot.out b/contrib/test_decoding/expected/slot.out index 63a9940f73..1c055f329c 100644 --- a/contrib/test_decoding/expected/slot.out +++ b/contrib/test_decoding/expected/slot.out @@ -406,3 +406,22 @@ SELECT pg_drop_replication_slot('copied_slot2_notemp'); (1 row) +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); + ?column? +---------- + init +(1 row) + +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + slot_name | slot_type | failover +---------------+-----------+---------- + failover_slot | logical | t +(1 row) + +SELECT pg_drop_replication_slot('failover_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/slot.sql b/contrib/test_decoding/sql/slot.sql index 1aa27c5667..1133e45abb 100644 --- a/contrib/test_decoding/sql/slot.sql +++ b/contrib/test_decoding/sql/slot.sql @@ -176,3 +176,9 @@ ORDER BY o.slot_name, c.slot_name; SELECT pg_drop_replication_slot('orig_slot2'); SELECT pg_drop_replication_slot('copied_slot2_no_change'); SELECT pg_drop_replication_slot('copied_slot2_notemp'); + +-- Test logical slots creation with 'failover'=true (last arg) +SELECT 'init' FROM pg_create_logical_replication_slot('failover_slot', 'test_decoding', false, false, true); +SELECT slot_name, slot_type, failover FROM pg_replication_slots; + +SELECT pg_drop_replication_slot('failover_slot'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3ec7391ec5..e666730c64 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7990,6 +7990,18 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l </para></entry> </row> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>subfailoverstate</structfield> <type>char</type> + </para> + <para> + State codes for failover mode: + <literal>d</literal> = disabled, + <literal>p</literal> = pending enablement, + <literal>e</literal> = enabled + </para></entry> + </row> + <row> <entry role="catalog_table_entry"><para role="column_definition"> <structfield>subconninfo</structfield> <type>text</type> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 93735e3aea..e8fbe8ca55 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4344,6 +4344,28 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </listitem> </varlistentry> + <varlistentry id="guc-standby-slot-names" xreflabel="standby_slot_names"> + <term><varname>standby_slot_names</varname> (<type>string</type>) + <indexterm> + <primary><varname>standby_slot_names</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + List of physical replication slots that logical replication slots with + failover enabled waits for. If a logical replication connection is + meant to switch to a physical standby after the standby is promoted, + the physical replication slot for the standby should be listed here. + </para> + <para> + The standbys corresponding to the physical replication slots in + <varname>standby_slot_names</varname> must enable + <varname>enable_syncslot</varname> for the standbys to receive + failover logical slots changes from the primary. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 93f068edcf..7fb55ae444 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27535,7 +27535,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset <indexterm> <primary>pg_create_logical_replication_slot</primary> </indexterm> - <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type> </optional> ) + <function>pg_create_logical_replication_slot</function> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>plugin</parameter> <type>name</type> <optional>, <parameter>temporary</parameter> <type>boolean</type>, <parameter>twophase</parameter> <type>boolean</type>, <parameter>failover</parameter> <type>boolean</type> </optional> ) <returnvalue>record</returnvalue> ( <parameter>slot_name</parameter> <type>name</type>, <parameter>lsn</parameter> <type>pg_lsn</type> ) @@ -27550,8 +27550,13 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset released upon any error. The optional fourth parameter, <parameter>twophase</parameter>, when set to true, specifies that the decoding of prepared transactions is enabled for this - slot. A call to this function has the same effect as the replication - protocol command <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. + slot. The optional fifth parameter, + <parameter>failover</parameter>, when set to true, + specifies that this slot is enabled to be synced to the + physical standbys so that logical replication can be resumed + after failover. A call to this function has the same effect as + the replication protocol command + <literal>CREATE_REPLICATION_SLOT ... LOGICAL</literal>. </para></entry> </row> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..bb926ab149 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2060,6 +2060,16 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> </variablelist> <para> @@ -2124,6 +2134,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-replication-alter-replication-slot" xreflabel="ALTER_REPLICATION_SLOT"> + <term><literal>ALTER_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> ( <replaceable class="parameter">option</replaceable> [, ...] ) + <indexterm><primary>ALTER_REPLICATION_SLOT</primary></indexterm> + </term> + <listitem> + <para> + Change the definition of a replication slot. + See <xref linkend="streaming-replication-slots"/> for more about + replication slots. This command is currently only supported for logical + replication slots. + </para> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">slot_name</replaceable></term> + <listitem> + <para> + The name of the slot to alter. Must be a valid replication slot + name (see <xref linkend="streaming-replication-slots-manipulation"/>). + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>The following options are supported:</para> + + <variablelist> + <varlistentry> + <term><literal>FAILOVER { 'true' | 'false' }</literal></term> + <listitem> + <para> + If true, the slot is enabled to be synced to the physical + standbys so that logical replication can be resumed after failover. + </para> + </listitem> + </varlistentry> + </variablelist> + + </listitem> + </varlistentry> + <varlistentry id="protocol-replication-read-replication-slot"> <term><literal>READ_REPLICATION_SLOT</literal> <replaceable class="parameter">slot_name</replaceable> <indexterm><primary>READ_REPLICATION_SLOT</primary></indexterm> diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6d36ff0dc9..e4fad5c55c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -73,10 +73,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO < These commands also cannot be executed when the subscription has <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link> - commit enabled, unless + commit enabled or + <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link> + enabled, unless <link linkend="sql-createsubscription-params-with-copy-data"><literal>copy_data</literal></link> is <literal>false</literal>. See column <structfield>subtwophasestate</structfield> - of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + and <structfield>subfailoverstate</structfield> of + <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> to know the actual two-phase state. </para> </refsect1> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f1c20b3a46..c7b1d7b3c0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -399,6 +399,30 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl </para> </listitem> </varlistentry> + + <varlistentry id="sql-createsubscription-params-with-failover"> + <term><literal>failover</literal> (<type>boolean</type>)</term> + <listitem> + <para> + Specifies whether the replication slot assocaited with the subscription + is enabled to be synced to the physical standbys so that logical + replication can be resumed from the new primary after failover. + The default is <literal>false</literal>. + </para> + + <para> + The implementation of failover requires that replication + has successfully finished the initial table synchronization + phase. So even when <literal>failover</literal> is enabled for a + subscription, the internal failover state remains + temporarily <quote>pending</quote> until the initialization phase + completes. See column <structfield>subfailoverstate</structfield> + of <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link> + to know the actual failover state. + </para> + + </listitem> + </varlistentry> </variablelist></para> </listitem> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7078491c4c..7ea08942c4 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2532,6 +2532,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx invalidated). Always NULL for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>failover</structfield> <type>bool</type> + </para> + <para> + True if this logical slot is enabled to be synced to the physical + standbys so that logical replication can be resumed from the new primary + after failover. Always false for physical slots. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index d6a978f136..18512955ad 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -73,6 +73,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->disableonerr = subform->subdisableonerr; sub->passwordrequired = subform->subpasswordrequired; sub->runasowner = subform->subrunasowner; + sub->failoverstate = subform->subfailoverstate; /* Get conninfo */ datum = SysCacheGetAttrNotNull(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 4206752881..4db796aa0b 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -479,6 +479,7 @@ CREATE OR REPLACE FUNCTION pg_create_logical_replication_slot( IN slot_name name, IN plugin name, IN temporary boolean DEFAULT false, IN twophase boolean DEFAULT false, + IN failover boolean DEFAULT false, OUT slot_name name, OUT lsn pg_lsn) RETURNS RECORD LANGUAGE INTERNAL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b65f6b5249..798c8d705d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1002,7 +1002,8 @@ CREATE VIEW pg_replication_slots AS L.wal_status, L.safe_wal_size, L.two_phase, - L.conflicting + L.conflicting, + L.failover FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); @@ -1333,7 +1334,8 @@ REVOKE ALL ON pg_subscription FROM public; GRANT SELECT (oid, subdbid, subskiplsn, subname, subowner, subenabled, subbinary, substream, subtwophasestate, subdisableonerr, subpasswordrequired, subrunasowner, - subslotname, subsynccommit, subpublications, suborigin) + subslotname, subsynccommit, subpublications, suborigin, + subfailoverstate) ON pg_subscription TO public; CREATE VIEW pg_stat_subscription_stats AS diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index edc82c11be..4f4cedc1fd 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -71,6 +71,7 @@ #define SUBOPT_RUN_AS_OWNER 0x00001000 #define SUBOPT_LSN 0x00002000 #define SUBOPT_ORIGIN 0x00004000 +#define SUBOPT_FAILOVER 0x00008000 /* check if the 'val' has 'bits' set */ #define IsSet(val, bits) (((val) & (bits)) == (bits)) @@ -96,6 +97,7 @@ typedef struct SubOpts bool passwordrequired; bool runasowner; char *origin; + bool failover; XLogRecPtr lsn; } SubOpts; @@ -157,6 +159,8 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, opts->runasowner = false; if (IsSet(supported_opts, SUBOPT_ORIGIN)) opts->origin = pstrdup(LOGICALREP_ORIGIN_ANY); + if (IsSet(supported_opts, SUBOPT_FAILOVER)) + opts->failover = false; /* Parse options */ foreach(lc, stmt_options) @@ -326,6 +330,15 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized origin value: \"%s\"", opts->origin)); } + else if (IsSet(supported_opts, SUBOPT_FAILOVER) && + strcmp(defel->defname, "failover") == 0) + { + if (IsSet(opts->specified_opts, SUBOPT_FAILOVER)) + errorConflictingDefElem(defel, pstate); + + opts->specified_opts |= SUBOPT_FAILOVER; + opts->failover = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_LSN) && strcmp(defel->defname, "lsn") == 0) { @@ -591,7 +604,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_STREAMING | SUBOPT_TWOPHASE_COMMIT | SUBOPT_DISABLE_ON_ERR | SUBOPT_PASSWORD_REQUIRED | - SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN); + SUBOPT_RUN_AS_OWNER | SUBOPT_ORIGIN | + SUBOPT_FAILOVER); parse_subscription_options(pstate, stmt->options, supported_opts, &opts); /* @@ -710,6 +724,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, publicationListToArray(publications); values[Anum_pg_subscription_suborigin - 1] = CStringGetTextDatum(opts.origin); + values[Anum_pg_subscription_subfailoverstate - 1] = + CharGetDatum(opts.failover ? + LOGICALREP_FAILOVER_STATE_PENDING : + LOGICALREP_FAILOVER_STATE_DISABLED); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -746,6 +764,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool failover_enabled = false; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -776,6 +796,19 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, InvalidXLogRecPtr); } + /* + * Even if failover is set, don't create the slot with failover + * enabled. Will enable it once all the tables are synced and + * ready. The intention is that if failover happens at the time of + * table-sync, user should re-launch the subscription instead of + * relying on main slot (if synced) with no table-sync data + * present. When the subscription has no tables, leave failover as + * false to allow ALTER SUBSCRIPTION ... REFRESH PUBLICATION to + * work. + */ + if (opts.failover && !opts.copy_data && tables != NIL) + failover_enabled = true; + /* * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to @@ -807,15 +840,32 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, - CRS_NOEXPORT_SNAPSHOT, NULL); - - if (twophase_enabled) - UpdateTwoPhaseState(subid, LOGICALREP_TWOPHASE_STATE_ENABLED); - + failover_enabled, CRS_NOEXPORT_SNAPSHOT, NULL); + + /* Update twophase and/or failover state */ + if (twophase_enabled || failover_enabled) + UpdateTwoPhaseFailoverStates(subid, + twophase_enabled, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_enabled, + LOGICALREP_FAILOVER_STATE_ENABLED); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", opts.slot_name))); } + + /* + * If only the slot_name is specified, it is possible that the user intends to + * use an existing slot on the publisher, so here we enable failover for the + * slot if requested. + */ + else if (opts.slot_name && failover_enabled) + { + walrcv_alter_slot(wrconn, opts.slot_name, opts.failover); + ereport(NOTICE, + (errmsg("enabled failover for replication slot \"%s\" on publisher", + opts.slot_name))); + } } PG_FINALLY(); { @@ -1288,6 +1338,12 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Make sure refresh sees the new list of publications. */ @@ -1347,6 +1403,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... ADD PUBLICATION" : "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION with refresh and copy_data is not allowed when failover is enabled"), + /* translator: %s is an SQL ALTER command */ + errhint("Use %s with refresh = false, or with copy_data = false, or use DROP/CREATE SUBSCRIPTION.", + isadd ? + "ALTER SUBSCRIPTION ... ADD PUBLICATION" : + "ALTER SUBSCRIPTION ... DROP PUBLICATION"))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION with refresh"); /* Refresh the new list of publications. */ @@ -1392,6 +1458,16 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + /* + * See comments above for twophasestate, same holds true for + * 'failover' + */ + if (sub->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED && opts.copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when failover is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); AlterSubscription_refresh(sub, opts.copy_data, NULL); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 60d5c1fc40..336c2bec99 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -74,8 +74,11 @@ static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover); static pid_t libpqrcv_get_backend_pid(WalReceiverConn *conn); static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, const char *query, @@ -96,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_receive = libpqrcv_receive, .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, + .walrcv_alter_slot = libpqrcv_alter_slot, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -883,8 +887,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool two_phase, CRSSnapshotAction snapshot_action, - XLogRecPtr *lsn) + bool temporary, bool two_phase, bool failover, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -913,7 +917,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, else appendStringInfoChar(&cmd, ' '); } - + if (failover) + { + appendStringInfoString(&cmd, "FAILOVER"); + if (use_new_options_syntax) + appendStringInfoString(&cmd, ", "); + else + appendStringInfoChar(&cmd, ' '); + } if (use_new_options_syntax) { switch (snapshot_action) @@ -982,6 +993,33 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, return snapshot; } +/* + * Change the definition of the replication slot. + */ +static void +libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, + bool failover) +{ + StringInfoData cmd; + PGresult *res; + + initStringInfo(&cmd); + appendStringInfo(&cmd, "ALTER_REPLICATION_SLOT %s ( FAILOVER %s )", + quote_identifier(slotname), + failover ? "true" : "false"); + + res = libpqrcv_PQexec(conn->streamConn, cmd.data); + pfree(cmd.data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("could not alter replication slot \"%s\" on publisher: %s", + slotname, pchomp(PQerrorMessage(conn->streamConn))))); + + PQclear(res); +} + /* * Return PID of remote backend process. */ diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1067aca08f..cf22f7aa43 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -30,6 +30,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/message.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "utils/array.h" #include "utils/builtins.h" @@ -109,6 +110,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin MemoryContext per_query_ctx; MemoryContext oldcontext; XLogRecPtr end_of_wal; + XLogRecPtr wal_to_wait; LogicalDecodingContext *ctx; ResourceOwner old_resowner = CurrentResourceOwner; ArrayType *arr; @@ -228,6 +230,17 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin NameStr(MyReplicationSlot->data.plugin), format_procedure(fcinfo->flinfo->fn_oid)))); + if (XLogRecPtrIsInvalid(upto_lsn)) + wal_to_wait = end_of_wal; + else + wal_to_wait = Min(upto_lsn, end_of_wal); + + /* + * Wait for specified streaming replication standby servers (if any) + * to confirm receipt of WAL upto wal_to_wait. + */ + WalSndWaitForStandbyConfirmation(wal_to_wait); + ctx->output_writer_private = p; /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index df3c42eb5d..f99c12513d 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -614,15 +614,26 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Note: If the subscription has no tables then leave the state as * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. + * + * Same goes for 'failover'. Enable it only if subscription has tables + * and all the tablesyncs have reached READY state. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING || + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) { CommandCounterIncrement(); /* make updates visible */ if (AllTablesyncsReady()) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", - MySubscription->name))); + if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled", + MySubscription->name))); + + if (MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING) + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart so that failover can be enabled", + MySubscription->name))); + should_exit = true; } } @@ -1420,7 +1431,8 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) */ walrcv_create_slot(LogRepWorkerWalRcvConn, slotname, false /* permanent */ , false /* two_phase */ , - CRS_USE_SNAPSHOT, origin_startpos); + false /* failover */ , CRS_USE_SNAPSHOT, + origin_startpos); /* * Setup replication origin tracking. The purpose of doing this before the @@ -1722,10 +1734,13 @@ AllTablesyncsReady(void) } /* - * Update the two_phase state of the specified subscription in pg_subscription. + * Update the twophase and/or failover state of the specified subscription + * in pg_subscription. */ void -UpdateTwoPhaseState(Oid suboid, char new_state) +UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover) { Relation rel; HeapTuple tup; @@ -1733,9 +1748,15 @@ UpdateTwoPhaseState(Oid suboid, char new_state) bool replaces[Natts_pg_subscription]; Datum values[Natts_pg_subscription]; - Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED || - new_state == LOGICALREP_TWOPHASE_STATE_PENDING || - new_state == LOGICALREP_TWOPHASE_STATE_ENABLED); + if (update_twophase) + Assert(new_state_twophase == LOGICALREP_TWOPHASE_STATE_DISABLED || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_PENDING || + new_state_twophase == LOGICALREP_TWOPHASE_STATE_ENABLED); + + if (update_failover) + Assert(new_state_failover == LOGICALREP_FAILOVER_STATE_DISABLED || + new_state_failover == LOGICALREP_FAILOVER_STATE_PENDING || + new_state_failover == LOGICALREP_FAILOVER_STATE_ENABLED); rel = table_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid)); @@ -1749,9 +1770,19 @@ UpdateTwoPhaseState(Oid suboid, char new_state) memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - /* And update/set two_phase state */ - values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state); - replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + /* Update/set two_phase state if asked by the caller */ + if (update_twophase) + { + values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state_twophase); + replaces[Anum_pg_subscription_subtwophasestate - 1] = true; + } + + /* Update/set failover state if asked by the caller */ + if (update_failover) + { + values[Anum_pg_subscription_subfailoverstate - 1] = CharGetDatum(new_state_failover); + replaces[Anum_pg_subscription_subfailoverstate - 1] = true; + } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 21abf34ef7..4fdac7bc66 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3947,6 +3947,7 @@ maybe_reread_subscription(void) newsub->passwordrequired != MySubscription->passwordrequired || strcmp(newsub->origin, MySubscription->origin) != 0 || newsub->owner != MySubscription->owner || + newsub->failoverstate != MySubscription->failoverstate || !equal(newsub->publications, MySubscription->publications)) { if (am_parallel_apply_worker()) @@ -4482,6 +4483,8 @@ run_apply_worker() TimeLineID startpointTLI; char *err; bool must_use_password; + bool twophase_pending; + bool failover_pending; slotname = MySubscription->slotname; @@ -4539,16 +4542,37 @@ run_apply_worker() * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to * work. */ - if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING && - AllTablesyncsReady()) + twophase_pending = (MySubscription->twophasestate + == LOGICALREP_TWOPHASE_STATE_PENDING) ? true : false; + failover_pending = (MySubscription->failoverstate + == LOGICALREP_FAILOVER_STATE_PENDING) ? true : false; + + if ((twophase_pending || failover_pending) && AllTablesyncsReady()) { /* Start streaming with two_phase enabled */ - options.proto.logical.twophase = true; + if (twophase_pending) + options.proto.logical.twophase = true; + + if (failover_pending) + walrcv_alter_slot(LogRepWorkerWalRcvConn, slotname, true); + walrcv_startstreaming(LogRepWorkerWalRcvConn, &options); StartTransactionCommand(); - UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED); - MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + /* Update twophase and/or failover */ + if (twophase_pending || failover_pending) + UpdateTwoPhaseFailoverStates(MySubscription->oid, + twophase_pending, + LOGICALREP_TWOPHASE_STATE_ENABLED, + failover_pending, + LOGICALREP_FAILOVER_STATE_ENABLED); + if (twophase_pending) + MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED; + + if (failover_pending) + MySubscription->failoverstate = LOGICALREP_FAILOVER_STATE_ENABLED; + CommitTransactionCommand(); } else @@ -4557,11 +4581,15 @@ run_apply_worker() } ereport(DEBUG1, - (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s", + (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s and failover is %s", MySubscription->name, MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" : MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" : + "?", + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_DISABLED ? "DISABLED" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_PENDING ? "PENDING" : + MySubscription->failoverstate == LOGICALREP_FAILOVER_STATE_ENABLED ? "ENABLED" : "?"))); /* Run the main loop. */ diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 0c874e33cf..b706046811 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -64,6 +64,7 @@ Node *replication_parse_result; %token K_START_REPLICATION %token K_CREATE_REPLICATION_SLOT %token K_DROP_REPLICATION_SLOT +%token K_ALTER_REPLICATION_SLOT %token K_TIMELINE_HISTORY %token K_WAIT %token K_TIMELINE @@ -79,7 +80,8 @@ Node *replication_parse_result; %type <node> command %type <node> base_backup start_replication start_logical_replication - create_replication_slot drop_replication_slot identify_system + create_replication_slot drop_replication_slot + alter_replication_slot identify_system read_replication_slot timeline_history show %type <list> generic_option_list %type <defelt> generic_option @@ -111,6 +113,7 @@ command: | start_logical_replication | create_replication_slot | drop_replication_slot + | alter_replication_slot | read_replication_slot | timeline_history | show @@ -257,6 +260,18 @@ drop_replication_slot: } ; +/* ALTER_REPLICATION_SLOT slot */ +alter_replication_slot: + K_ALTER_REPLICATION_SLOT IDENT '(' generic_option_list ')' + { + AlterReplicationSlotCmd *cmd; + cmd = makeNode(AlterReplicationSlotCmd); + cmd->slotname = $2; + cmd->options = $4; + $$ = (Node *) cmd; + } + ; + /* * START_REPLICATION [SLOT slot] [PHYSICAL] %X/%X [TIMELINE %d] */ @@ -399,6 +414,7 @@ ident_or_keyword: | K_START_REPLICATION { $$ = "start_replication"; } | K_CREATE_REPLICATION_SLOT { $$ = "create_replication_slot"; } | K_DROP_REPLICATION_SLOT { $$ = "drop_replication_slot"; } + | K_ALTER_REPLICATION_SLOT { $$ = "alter_replication_slot"; } | K_TIMELINE_HISTORY { $$ = "timeline_history"; } | K_WAIT { $$ = "wait"; } | K_TIMELINE { $$ = "timeline"; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 1cc7fb858c..0b5ae23195 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -125,6 +125,7 @@ TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } DROP_REPLICATION_SLOT { return K_DROP_REPLICATION_SLOT; } +ALTER_REPLICATION_SLOT { return K_ALTER_REPLICATION_SLOT; } TIMELINE_HISTORY { return K_TIMELINE_HISTORY; } PHYSICAL { return K_PHYSICAL; } RESERVE_WAL { return K_RESERVE_WAL; } @@ -301,6 +302,7 @@ replication_scanner_is_replication_command(void) case K_START_REPLICATION: case K_CREATE_REPLICATION_SLOT: case K_DROP_REPLICATION_SLOT: + case K_ALTER_REPLICATION_SLOT: case K_READ_REPLICATION_SLOT: case K_TIMELINE_HISTORY: case K_SHOW: diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 18bc28195b..85dd935e65 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -52,6 +52,9 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/memutils.h" +#include "utils/varlena.h" /* * Replication slot on-disk data structure. @@ -90,7 +93,7 @@ typedef struct ReplicationSlotOnDisk sizeof(ReplicationSlotOnDisk) - ReplicationSlotOnDiskConstantSize #define SLOT_MAGIC 0x1051CA1 /* format identifier */ -#define SLOT_VERSION 3 /* version for new files */ +#define SLOT_VERSION 4 /* version for new files */ /* Control array for replication slot management */ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; @@ -98,9 +101,11 @@ ReplicationSlotCtlData *ReplicationSlotCtl = NULL; /* My backend's replication slot in the shared memory array */ ReplicationSlot *MyReplicationSlot = NULL; -/* GUC variable */ +/* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +char *standby_slot_names; +static List *standby_slot_names_list = NIL; static void ReplicationSlotShmemExit(int code, Datum arg); static void ReplicationSlotDropAcquired(void); @@ -251,7 +256,8 @@ ReplicationSlotValidateName(const char *name, int elevel) */ void ReplicationSlotCreate(const char *name, bool db_specific, - ReplicationSlotPersistency persistency, bool two_phase) + ReplicationSlotPersistency persistency, + bool two_phase, bool failover) { ReplicationSlot *slot = NULL; int i; @@ -311,6 +317,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.persistency = persistency; slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; + slot->data.failover = failover; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -679,6 +686,31 @@ ReplicationSlotDrop(const char *name, bool nowait) ReplicationSlotDropAcquired(); } +/* + * Change the definition of the slot identified by the passed in name. + */ +void +ReplicationSlotAlter(const char *name, bool failover) +{ + Assert(MyReplicationSlot == NULL); + + ReplicationSlotAcquire(name, true); + + if (SlotIsPhysical(MyReplicationSlot)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot use %s with a physical replication slot", + "ALTER_REPLICATION_SLOT")); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.failover = failover; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + ReplicationSlotRelease(); +} + /* * Permanently drop the currently acquired replication slot. */ @@ -2159,3 +2191,138 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * A helper function to validate slots specified in standby_slot_names GUCs. + */ +static bool +validate_standby_slots(char **newval) +{ + char *rawname; + List *elemlist; + ListCell *lc; + + /* Need a modifiable copy of string */ + rawname = pstrdup(*newval); + + /* Verify syntax and parse string into list of identifiers */ + if (!SplitIdentifierString(rawname, ',', &elemlist)) + { + /* syntax error in name list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawname); + list_free(elemlist); + return false; + } + + /* + * Verify 'type' of slot now. + * + * Skip check if replication slots' data is not initialized yet i.e. we + * are in startup process. + */ + if (!ReplicationSlotCtl) + return true; + + foreach(lc, elemlist) + { + char *name = lfirst(lc); + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (!slot) + { + GUC_check_errdetail("replication slot \"%s\" does not exist", name); + list_free(elemlist); + return false; + } + + if (SlotIsLogical(slot)) + { + GUC_check_errdetail("cannot have logical replication slot \"%s\" " + "in this parameter", name); + list_free(elemlist); + return false; + } + } + + list_free(elemlist); + return true; +} + +/* + * GUC check_hook for standby_slot_names + */ +bool +check_standby_slot_names(char **newval, void **extra, GucSource source) +{ + if (strcmp(*newval, "") == 0) + return true; + + /* + * "*" is not accepted as in that case primary will not be able to know + * for which all standbys to wait for. Even if we have physical-slots + * info, there is no way to confirm whether there is any standby + * configured for the known physical slots. + */ + if (strcmp(*newval, "*") == 0) + { + GUC_check_errdetail("\"%s\" is not accepted for standby_slot_names", + *newval); + return false; + } + + /* Now verify if the specified slots really exist and have correct type */ + if (!validate_standby_slots(newval)) + return false; + + *extra = guc_strdup(ERROR, *newval); + + return true; +} + +/* + * GUC assign_hook for standby_slot_names + */ +void +assign_standby_slot_names(const char *newval, void *extra) +{ + List *standby_slots; + MemoryContext oldcxt; + char *standby_slot_names_cpy = extra; + + list_free(standby_slot_names_list); + standby_slot_names_list = NIL; + + /* No value is specified for standby_slot_names. */ + if (standby_slot_names_cpy == NULL) + return; + + if (!SplitIdentifierString(standby_slot_names_cpy, ',', &standby_slots)) + { + /* This should not happen if GUC checked check_standby_slot_names. */ + elog(ERROR, "invalid list syntax"); + } + + /* + * Switch to the same memory context under which GUC variables are + * allocated (GUCMemoryContext). + */ + oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(standby_slot_names_cpy)); + standby_slot_names_list = list_copy(standby_slots); + MemoryContextSwitchTo(oldcxt); +} + +/* + * Return a copy of standby_slot_names_list if the copy flag is set to true, + * otherwise return the original list. + */ +List * +GetStandbySlotList(bool copy) +{ + if (copy) + return list_copy(standby_slot_names_list); + else + return standby_slot_names_list; +} diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4b694a03d0..fb6e37d2c3 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -21,6 +21,7 @@ #include "replication/decode.h" #include "replication/logical.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/pg_lsn.h" @@ -42,7 +43,8 @@ create_physical_replication_slot(char *name, bool immediately_reserve, /* acquire replication slot, this will check for conflicting names */ ReplicationSlotCreate(name, false, - temporary ? RS_TEMPORARY : RS_PERSISTENT, false); + temporary ? RS_TEMPORARY : RS_PERSISTENT, false, + false); if (immediately_reserve) { @@ -117,6 +119,7 @@ pg_create_physical_replication_slot(PG_FUNCTION_ARGS) static void create_logical_replication_slot(char *name, char *plugin, bool temporary, bool two_phase, + bool failover, XLogRecPtr restart_lsn, bool find_startpoint) { @@ -133,7 +136,8 @@ create_logical_replication_slot(char *name, char *plugin, * error as well. */ ReplicationSlotCreate(name, true, - temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase); + temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase, + failover); /* * Create logical decoding context to find start point or, if we don't @@ -171,6 +175,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) Name plugin = PG_GETARG_NAME(1); bool temporary = PG_GETARG_BOOL(2); bool two_phase = PG_GETARG_BOOL(3); + bool failover = PG_GETARG_BOOL(4); Datum result; TupleDesc tupdesc; HeapTuple tuple; @@ -188,6 +193,7 @@ pg_create_logical_replication_slot(PG_FUNCTION_ARGS) NameStr(*plugin), temporary, two_phase, + failover, InvalidXLogRecPtr, true); @@ -232,7 +238,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 15 +#define PG_GET_REPLICATION_SLOTS_COLS 16 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -412,6 +418,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(false); } + values[i++] = BoolGetDatum(slot_contents.data.failover); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -451,6 +459,8 @@ pg_physical_replication_slot_advance(XLogRecPtr moveto) * crash, but this makes the data consistent after a clean shutdown. */ ReplicationSlotMarkDirty(); + + PhysicalWakeupLogicalWalSnd(); } return retlsn; @@ -679,6 +689,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) XLogRecPtr src_restart_lsn; bool src_islogical; bool temporary; + bool failover; char *plugin; Datum values[2]; bool nulls[2]; @@ -734,6 +745,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) src_islogical = SlotIsLogical(&first_slot_contents); src_restart_lsn = first_slot_contents.data.restart_lsn; temporary = (first_slot_contents.data.persistency == RS_TEMPORARY); + failover = first_slot_contents.data.failover; plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL; /* Check type of replication slot */ @@ -773,6 +785,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot) plugin, temporary, false, + failover, src_restart_lsn, false); } diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2398167f49..e27d231174 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -386,7 +386,7 @@ WalReceiverMain(void) "pg_walreceiver_%lld", (long long int) walrcv_get_backend_pid(wrconn)); - walrcv_create_slot(wrconn, slotname, true, false, 0, NULL); + walrcv_create_slot(wrconn, slotname, true, false, false, 0, NULL); SpinLockAcquire(&walrcv->mutex); strlcpy(walrcv->slotname, slotname, NAMEDATALEN); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 3bc9c82389..6c062eef16 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -247,7 +247,8 @@ static void WalSndKeepalive(bool requestReply, XLogRecPtr writePtr); static void WalSndKeepaliveIfNecessary(void); static void WalSndCheckTimeOut(void); static long WalSndComputeSleeptime(TimestampTz now); -static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event); +static void WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, @@ -974,12 +975,13 @@ static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, CRSSnapshotAction *snapshot_action, - bool *two_phase) + bool *two_phase, bool *failover) { ListCell *lc; bool snapshot_action_given = false; bool reserve_wal_given = false; bool two_phase_given = false; + bool failover_given = false; /* Parse options */ foreach(lc, cmd->options) @@ -1029,6 +1031,15 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, two_phase_given = true; *two_phase = defGetBoolean(defel); } + else if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -1045,6 +1056,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char *slot_name; bool reserve_wal = false; bool two_phase = false; + bool failover = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; @@ -1054,13 +1066,13 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase); - + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action, &two_phase, + &failover); if (cmd->kind == REPLICATION_KIND_PHYSICAL) { ReplicationSlotCreate(cmd->slotname, false, cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT, - false); + false, false); if (reserve_wal) { @@ -1091,7 +1103,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) */ ReplicationSlotCreate(cmd->slotname, true, cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL, - two_phase); + two_phase, failover); /* * Do options check early so that we can bail before calling the @@ -1246,6 +1258,46 @@ DropReplicationSlot(DropReplicationSlotCmd *cmd) ReplicationSlotDrop(cmd->slotname, !cmd->wait); } +/* + * Process extra options given to ALTER_REPLICATION_SLOT. + */ +static void +parseAlterReplSlotOptions(AlterReplicationSlotCmd *cmd, bool *failover) +{ + ListCell *lc; + bool failover_given = false; + + /* Parse options */ + foreach(lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "failover") == 0) + { + if (failover_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + failover_given = true; + *failover = defGetBoolean(defel); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + +/* + * Change the definition of a replication slot. + */ +static void +AlterReplicationSlot(AlterReplicationSlotCmd *cmd) +{ + bool failover = false; + + parseAlterReplSlotOptions(cmd, &failover); + ReplicationSlotAlter(cmd->slotname, failover); +} + /* * Load previously initiated logical slot and prepare for sending data (via * WalSndLoop). @@ -1435,7 +1487,7 @@ ProcessPendingWrites(void) /* Sleep until something happens or we time out */ WalSndWait(WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE, sleeptime, - WAIT_EVENT_WAL_SENDER_WRITE_DATA); + WAIT_EVENT_WAL_SENDER_WRITE_DATA, false); /* Clear any already-pending wakeups */ ResetLatch(MyLatch); @@ -1527,27 +1579,238 @@ WalSndUpdateProgress(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId ProcessPendingWrites(); } +/* + * Wake up logical walsenders with failover-enabled slots if the physical slot + * of the current walsender is specified in standby_slot_names GUC. + */ +void +PhysicalWakeupLogicalWalSnd(void) +{ + ListCell *lc; + List *standby_slots; + bool slot_in_list = false; + + Assert(MyReplicationSlot != NULL); + Assert(SlotIsPhysical(MyReplicationSlot)); + + standby_slots = GetStandbySlotList(false); + + foreach(lc, standby_slots) + { + char *name = lfirst(lc); + + if (strcmp(name, NameStr(MyReplicationSlot->data.name)) == 0) + { + slot_in_list = true; + break; + } + } + + if (slot_in_list) + ConditionVariableBroadcast(&WalSndCtl->wal_confirm_rcv_cv); +} + +/* + * Reload the config file and reinitialize the standby slot list if the GUC + * standby_slot_names has changed. + */ +static void +WalSndRereadConfigAndReInitSlotList(List **standby_slots) +{ + char *pre_standby_slot_names = pstrdup(standby_slot_names); + + ProcessConfigFile(PGC_SIGHUP); + + if (strcmp(pre_standby_slot_names, standby_slot_names) != 0) + { + list_free(*standby_slots); + *standby_slots = GetStandbySlotList(true); + } + + pfree(pre_standby_slot_names); +} + +/* + * Filter the standby slots based on the specified log sequence number + * (wait_for_lsn). + * + * This function updates the passed standby_slots list, removing any slots that + * have already caught up to or surpassed the given wait_for_lsn. + */ +static void +WalSndFilterStandbySlots(XLogRecPtr wait_for_lsn, List **standby_slots) +{ + ListCell *lc; + List *standby_slots_cpy = *standby_slots; + + foreach(lc, standby_slots_cpy) + { + char *name = lfirst(lc); + XLogRecPtr restart_lsn = InvalidXLogRecPtr; + bool invalidated = false; + char *warningfmt = NULL; + ReplicationSlot *slot; + + slot = SearchNamedReplicationSlot(name, true); + + if (slot && SlotIsPhysical(slot)) + { + SpinLockAcquire(&slot->mutex); + restart_lsn = slot->data.restart_lsn; + invalidated = slot->data.invalidated != RS_INVAL_NONE; + SpinLockRelease(&slot->mutex); + } + + /* Continue if the current slot hasn't caught up. */ + if (!invalidated && !XLogRecPtrIsInvalid(restart_lsn) && + restart_lsn < wait_for_lsn) + { + /* Log warning if no active_pid for this physical slot */ + if (slot->active_pid == 0) + ereport(WARNING, + errmsg("replication slot \"%s\" specified in parameter \"%s\" does not have active_pid", + name, "standby_slot_names"), + errdetail("Logical replication is waiting on the " + "standby associated with \"%s\"", name), + errhint("Consider starting standby associated with " + "\"%s\" or amend standby_slot_names", name)); + + continue; + } + + /* + * It may happen that the slot specified in standby_slot_names GUC + * value is dropped, so let's skip over it. + */ + else if (!slot) + warningfmt = _("replication slot \"%s\" specified in parameter \"%s\" does not exist, ignoring"); + + /* + * If logical slot name is given in standby_slot_names, give WARNING + * and skip it. Since it is harmless, so WARNING should be enough, no + * need to error-out. + */ + else if (SlotIsLogical(slot)) + warningfmt = _("cannot have logical replication slot \"%s\" in parameter \"%s\", ignoring"); + + /* + * Specified physical slot may have been invalidated, so no point in + * waiting for it. + */ + else if (XLogRecPtrIsInvalid(restart_lsn) || invalidated) + warningfmt = _("physical slot \"%s\" specified in parameter \"%s\" has been invalidated, ignoring"); + else + Assert(restart_lsn >= wait_for_lsn); + + /* + * Reaching here indicates that either the slot has passed the + * wait_for_lsn or there is an issue with the slot that requires a + * warning to be reported. + */ + if (warningfmt) + ereport(WARNING, errmsg(warningfmt, name, "standby_slot_names")); + + standby_slots_cpy = foreach_delete_current(standby_slots_cpy, lc); + } + + *standby_slots = standby_slots_cpy; +} + +/* + * Wait for physical standby to confirm receiving given lsn. + * + * Used by logical decoding SQL functions that acquired slot with failover + * enabled. It waits for physical standbys corresponding to the physical slots + * specified in the standby_slot_names GUC. + */ +void +WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) +{ + List *standby_slots; + + Assert(!am_walsender); + + if (!MyReplicationSlot->data.failover) + return; + + standby_slots = GetStandbySlotList(true); + + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + + for (;;) + { + long sleeptime = -1; + + WalSndFilterStandbySlots(wait_for_lsn, &standby_slots); + + /* Exit if done waiting for every slot. */ + if (standby_slots == NIL) + break; + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + WalSndRereadConfigAndReInitSlotList(&standby_slots); + } + + sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp()); + + ConditionVariableTimedSleep(&WalSndCtl->wal_confirm_rcv_cv, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION); + } + + ConditionVariableCancelSleep(); + list_free(standby_slots); +} + /* * Wait till WAL < loc is flushed to disk so it can be safely sent to client. * - * Returns end LSN of flushed WAL. Normally this will be >= loc, but - * if we detect a shutdown request (either from postmaster or client) - * we will return early, so caller must always check. + * If the walsender holds a logical slot that has enabled failover, the + * function also waits for all the specified streaming replication standby + * servers to confirm receipt of WAL upto RecentFlushPtr. + * + * Returns end LSN of flushed WAL. Normally this will be >= loc, but if we + * detect a shutdown request (either from postmaster or client) we will return + * early, so caller must always check. */ static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc) { int wakeEvents; + bool wait_for_standby = false; + uint32 wait_event; + List *standby_slots = NIL; static XLogRecPtr RecentFlushPtr = InvalidXLogRecPtr; + if (MyReplicationSlot->data.failover) + standby_slots = GetStandbySlotList(true); + /* - * Fast path to avoid acquiring the spinlock in case we already know we - * have enough WAL available. This is particularly interesting if we're - * far behind. + * Check if all the standby servers have confirmed receipt of WAL upto + * RecentFlushPtr if we already know we have enough WAL available. + * + * Note that we cannot directly return without checking the status of + * standby servers because the standby_slot_names may have changed, which + * means there could be new standby slots in the list that have not yet + * caught up to the RecentFlushPtr. */ if (RecentFlushPtr != InvalidXLogRecPtr && loc <= RecentFlushPtr) - return RecentFlushPtr; + { + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + + /* + * Fast path to avoid acquiring the spinlock in case we already know we + * have enough WAL available and all the standby servers has confirmed + * receipt of WAL upto RecentFlushPtr. This is particularly interesting + * if we're far behind. + */ + if (standby_slots == NIL) + return RecentFlushPtr; + } /* Get a more recent flush pointer. */ if (!RecoveryInProgress()) @@ -1568,7 +1831,7 @@ WalSndWaitForWal(XLogRecPtr loc) if (ConfigReloadPending) { ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); + WalSndRereadConfigAndReInitSlotList(&standby_slots); SyncRepInitConfig(); } @@ -1583,8 +1846,18 @@ WalSndWaitForWal(XLogRecPtr loc) if (got_STOPPING) XLogBackgroundFlush(); + /* + * Update the standby slots that have not yet caught up to the flushed + * position. It is good to wait upto RecentFlushPtr and then let it + * send the changes to logical subscribers one by one which are + * already covered in RecentFlushPtr without needing to wait on every + * change for standby confirmation. + */ + if (wait_for_standby) + WalSndFilterStandbySlots(RecentFlushPtr, &standby_slots); + /* Update our idea of the currently flushed position. */ - if (!RecoveryInProgress()) + else if (!RecoveryInProgress()) RecentFlushPtr = GetFlushRecPtr(NULL); else RecentFlushPtr = GetXLogReplayRecPtr(NULL); @@ -1612,8 +1885,14 @@ WalSndWaitForWal(XLogRecPtr loc) !waiting_for_ping_response) WalSndKeepalive(false, InvalidXLogRecPtr); - /* check whether we're done */ - if (loc <= RecentFlushPtr) + if (loc > RecentFlushPtr) + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL; + else if (standby_slots) + { + wait_event = WAIT_EVENT_WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION; + wait_for_standby = true; + } + else break; /* Waiting for new WAL. Since we need to wait, we're now caught up. */ @@ -1654,9 +1933,11 @@ WalSndWaitForWal(XLogRecPtr loc) if (pq_is_send_pending()) wakeEvents |= WL_SOCKET_WRITEABLE; - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_WAIT_FOR_WAL); + WalSndWait(wakeEvents, sleeptime, wait_event, wait_for_standby); } + list_free(standby_slots); + /* reactivate latch so WalSndLoop knows to continue */ SetLatch(MyLatch); return RecentFlushPtr; @@ -1819,6 +2100,13 @@ exec_replication_command(const char *cmd_string) EndReplicationCommand(cmdtag); break; + case T_AlterReplicationSlotCmd: + cmdtag = "ALTER_REPLICATION_SLOT"; + set_ps_display(cmdtag); + AlterReplicationSlot((AlterReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); + break; + case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; @@ -2049,6 +2337,7 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) { ReplicationSlotMarkDirty(); ReplicationSlotsComputeRequiredLSN(); + PhysicalWakeupLogicalWalSnd(); } /* @@ -2562,7 +2851,8 @@ WalSndLoop(WalSndSendDataCallback send_data) wakeEvents |= WL_SOCKET_WRITEABLE; /* Sleep until something happens or we time out */ - WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN); + WalSndWait(wakeEvents, sleeptime, WAIT_EVENT_WAL_SENDER_MAIN, + false); } } } @@ -3311,6 +3601,8 @@ WalSndShmemInit(void) ConditionVariableInit(&WalSndCtl->wal_flush_cv); ConditionVariableInit(&WalSndCtl->wal_replay_cv); + + ConditionVariableInit(&WalSndCtl->wal_confirm_rcv_cv); } } @@ -3351,7 +3643,8 @@ WalSndWakeup(bool physical, bool logical) * on postmaster death. */ static void -WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) +WalSndWait(uint32 socket_events, long timeout, uint32 wait_event, + bool wait_for_standby) { WaitEvent event; @@ -3380,8 +3673,14 @@ WalSndWait(uint32 socket_events, long timeout, uint32 wait_event) * * And, we use separate shared memory CVs for physical and logical * walsenders for selective wake ups, see WalSndWakeup() for more details. + * + * When the wait_for_standby flag is set to true, wait on another CV that + * is woken up by physical walsenders when the walreceiver has confirmed + * the receipt of LSN. */ - if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) + if (wait_for_standby) + ConditionVariablePrepareToSleep(&WalSndCtl->wal_confirm_rcv_cv); + else if (MyWalSnd->kind == REPLICATION_KIND_PHYSICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_flush_cv); else if (MyWalSnd->kind == REPLICATION_KIND_LOGICAL) ConditionVariablePrepareToSleep(&WalSndCtl->wal_replay_cv); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index d7995931bd..ede94a1ede 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -76,6 +76,7 @@ LIBPQWALRECEIVER_CONNECT "Waiting in WAL receiver to establish connection to rem LIBPQWALRECEIVER_RECEIVE "Waiting in WAL receiver to receive data from remote server." SSL_OPEN_SERVER "Waiting for SSL while attempting connection." WAL_SENDER_WAIT_FOR_WAL "Waiting for WAL to be flushed in WAL sender process." +WAL_SENDER_WAIT_FOR_STANDBY_CONFIRMATION "Waiting for the WAL to be received by physical standby in WAL sender process." WAL_SENDER_WRITE_DATA "Waiting for any activity when processing replies from WAL receiver in WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b764ef6998..be05790ff3 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4562,6 +4562,20 @@ struct config_string ConfigureNamesString[] = check_debug_io_direct, assign_debug_io_direct, NULL }, + { + {"standby_slot_names", PGC_SIGHUP, REPLICATION_PRIMARY, + gettext_noop("Lists streaming replication standby server slot " + "names that logical WAL sender processes wait for."), + gettext_noop("Decoded changes are sent out to plugins by logical " + "WAL sender processes only after specified " + "replication slots confirm receiving WAL."), + GUC_LIST_INPUT | GUC_LIST_QUOTE + }, + &standby_slot_names, + "", + check_standby_slot_names, assign_standby_slot_names, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e48c066a5b..dd2769cdd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -326,6 +326,8 @@ # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all +#standby_slot_names = '' # streaming replication standby server slot names that + # logical walsenders waits for # - Standby Servers - diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 4878aa22bf..5db7c06164 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -661,7 +661,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) * started and stopped several times causing any temporary slots to be * removed. */ - res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, " + res = executeQueryOrDie(conn, "SELECT slot_name, plugin, two_phase, failover, " "%s as caught_up, conflicting as invalid " "FROM pg_catalog.pg_replication_slots " "WHERE slot_type = 'logical' AND " @@ -681,6 +681,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) int i_twophase; int i_caught_up; int i_invalid; + int i_failover; slotinfos = (LogicalSlotInfo *) pg_malloc(sizeof(LogicalSlotInfo) * num_slots); @@ -689,6 +690,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) i_twophase = PQfnumber(res, "two_phase"); i_caught_up = PQfnumber(res, "caught_up"); i_invalid = PQfnumber(res, "invalid"); + i_failover = PQfnumber(res, "failover"); for (int slotnum = 0; slotnum < num_slots; slotnum++) { @@ -699,6 +701,7 @@ get_old_cluster_logical_slot_infos(DbInfo *dbinfo, bool live_check) curr->two_phase = (strcmp(PQgetvalue(res, slotnum, i_twophase), "t") == 0); curr->caught_up = (strcmp(PQgetvalue(res, slotnum, i_caught_up), "t") == 0); curr->invalid = (strcmp(PQgetvalue(res, slotnum, i_invalid), "t") == 0); + curr->failover = (strcmp(PQgetvalue(res, slotnum, i_failover), "t") == 0); } } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 3960af4036..90bed8ef72 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -916,8 +916,14 @@ create_logical_replication_slots(void) appendStringLiteralConn(query, slot_info->slotname, conn); appendPQExpBuffer(query, ", "); appendStringLiteralConn(query, slot_info->plugin, conn); - appendPQExpBuffer(query, ", false, %s);", - slot_info->two_phase ? "true" : "false"); + + if (GET_MAJOR_VERSION(new_cluster.major_version) >= 1700) + appendPQExpBuffer(query, ", false, %s, %s);", + slot_info->two_phase ? "true" : "false", + slot_info->failover ? "true" : "false"); + else + appendPQExpBuffer(query, ", false, %s);", + slot_info->two_phase ? "true" : "false"); PQclear(executeQueryOrDie(conn, "%s", query->data)); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index a710f325de..d47e950b77 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -160,6 +160,8 @@ typedef struct bool two_phase; /* can the slot decode 2PC? */ bool caught_up; /* has the slot caught up to latest changes? */ bool invalid; /* if true, the slot is unusable */ + bool failover; /* is the slot designated to be synced + * to the physical standby */ } LogicalSlotInfo; typedef struct diff --git a/src/bin/pg_upgrade/t/003_logical_slots.pl b/src/bin/pg_upgrade/t/003_logical_slots.pl index 5b01cf8c40..0a1c467ed0 100644 --- a/src/bin/pg_upgrade/t/003_logical_slots.pl +++ b/src/bin/pg_upgrade/t/003_logical_slots.pl @@ -158,7 +158,7 @@ $sub->start; $sub->safe_psql( 'postgres', qq[ CREATE TABLE tbl (a int); - CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true') + CREATE SUBSCRIPTION regress_sub CONNECTION '$old_connstr' PUBLICATION regress_pub WITH (two_phase = 'true', failover = 'true') ]); $sub->wait_for_subscription_sync($oldpub, 'regress_sub'); @@ -172,8 +172,8 @@ command_ok([@pg_upgrade_cmd], 'run of pg_upgrade of old cluster'); # Check that the slot 'regress_sub' has migrated to the new cluster $newpub->start; my $result = $newpub->safe_psql('postgres', - "SELECT slot_name, two_phase FROM pg_replication_slots"); -is($result, qq(regress_sub|t), 'check the slot exists on new cluster'); + "SELECT slot_name, two_phase, failover FROM pg_replication_slots"); +is($result, qq(regress_sub|t|t), 'check the slot exists on new cluster'); # Update the connection my $new_connstr = $newpub->connstr . ' dbname=postgres'; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 5077e7b358..36795b1085 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6563,7 +6563,8 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false, false, false, false, false, false, false, false}; + false, false, false, false, false, false, false, false, false, false, + false}; if (pset.sversion < 100000) { @@ -6627,6 +6628,11 @@ describeSubscriptions(const char *pattern, bool verbose) gettext_noop("Password required"), gettext_noop("Run as owner?")); + if (pset.sversion >= 170000) + appendPQExpBuffer(&buf, + ", subfailoverstate AS \"%s\"\n", + gettext_noop("Failover")); + appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" ", subconninfo AS \"%s\"\n", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 006e10f5d2..7634c86262 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3308,7 +3308,8 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("binary", "connect", "copy_data", "create_slot", "disable_on_error", "enabled", "origin", "password_required", "run_as_owner", "slot_name", - "streaming", "synchronous_commit", "two_phase"); + "streaming", "synchronous_commit", "two_phase", + "failover"); /* CREATE TRIGGER --- is allowed inside CREATE SCHEMA, so use TailMatches */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fb58dee3bc..d906734750 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11100,17 +11100,17 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'record', - proargtypes => 'name name bool bool', - proallargtypes => '{name,name,bool,bool,name,pg_lsn}', - proargmodes => '{i,i,i,i,o,o}', - proargnames => '{slot_name,plugin,temporary,twophase,slot_name,lsn}', + proargtypes => 'name name bool bool bool', + proallargtypes => '{name,name,bool,bool,bool,name,pg_lsn}', + proargmodes => '{i,i,i,i,i,o,o}', + proargnames => '{slot_name,plugin,temporary,twophase,failover,slot_name,lsn}', prosrc => 'pg_create_logical_replication_slot' }, { oid => '4222', descr => 'copy a logical replication slot, changing temporality and plugin', diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index e0b91eacd2..3f656b2f77 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -31,6 +31,10 @@ #define LOGICALREP_TWOPHASE_STATE_PENDING 'p' #define LOGICALREP_TWOPHASE_STATE_ENABLED 'e' +#define LOGICALREP_FAILOVER_STATE_DISABLED 'd' +#define LOGICALREP_FAILOVER_STATE_PENDING 'p' +#define LOGICALREP_FAILOVER_STATE_ENABLED 'e' + /* * The subscription will request the publisher to only send changes that do not * have any origin. @@ -93,6 +97,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subrunasowner; /* True if replication should execute as the * subscription owner */ + char subfailoverstate; /* Enable Failover State */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -145,6 +151,7 @@ typedef struct Subscription List *publications; /* List of publication names to subscribe to */ char *origin; /* Only publish data originating from the * specified origin */ + char failoverstate; /* Allow slot to be synchronized for failover */ } Subscription; /* Disallow streaming in-progress transactions. */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 5142a08729..bef8a7162e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -72,6 +72,18 @@ typedef struct DropReplicationSlotCmd } DropReplicationSlotCmd; +/* ---------------------- + * ALTER_REPLICATION_SLOT command + * ---------------------- + */ +typedef struct AlterReplicationSlotCmd +{ + NodeTag type; + char *slotname; + List *options; +} AlterReplicationSlotCmd; + + /* ---------------------- * START_REPLICATION command * ---------------------- diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index d3535eed58..a92fb38ec0 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -111,6 +111,12 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + + /* + * Is this a failover slot (sync candidate for physical standbys)? + * Relevant for logical slots on the primary server. + */ + bool failover; } ReplicationSlotPersistentData; /* @@ -210,6 +216,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; +extern PGDLLIMPORT char *standby_slot_names; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); @@ -218,9 +225,10 @@ extern void ReplicationSlotsShmemInit(void); /* management of individual slots */ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, - bool two_phase); + bool two_phase, bool failover); extern void ReplicationSlotPersist(void); extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); extern void ReplicationSlotRelease(void); @@ -253,4 +261,7 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); +extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); +extern List *GetStandbySlotList(bool copy); + #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 04b439dc50..115344f1c4 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -356,9 +356,20 @@ typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, bool two_phase, + bool failover, CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); +/* + * walrcv_alter_slot_fn + * + * Change the definition of a replication slot. Currently, it only supports + * changing the failover property of the slot. + */ +typedef void (*walrcv_alter_slot_fn) (WalReceiverConn *conn, + const char *slotname, + bool failover); + /* * walrcv_get_backend_pid_fn * @@ -400,6 +411,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; + walrcv_alter_slot_fn walrcv_alter_slot; walrcv_get_backend_pid_fn walrcv_get_backend_pid; walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; @@ -429,8 +441,10 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, snapshot_action, lsn) +#define walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, two_phase, failover, snapshot_action, lsn) +#define walrcv_alter_slot(conn, slotname, failover) \ + WalReceiverFunctions->walrcv_alter_slot(conn, slotname, failover) #define walrcv_get_backend_pid(conn) \ WalReceiverFunctions->walrcv_get_backend_pid(conn) #define walrcv_exec(conn, exec, nRetTypes, retTypes) \ diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 268f8e8d0f..1fcc22a127 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -14,6 +14,8 @@ #include <signal.h> +#include "access/xlogdefs.h" + /* * What to do with a snapshot in create replication slot command. */ @@ -47,6 +49,8 @@ extern void WalSndInitStopping(void); extern void WalSndWaitStopping(void); extern void HandleWalSndInitStopping(void); extern void WalSndRqstFileReload(void); +extern void PhysicalWakeupLogicalWalSnd(void); +extern void WalSndWaitForStandbyConfirmation(XLogRecPtr wait_for_lsn); /* * Remember that we want to wakeup walsenders later diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 13fd5877a6..48c6a7a146 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -113,6 +113,13 @@ typedef struct ConditionVariable wal_flush_cv; ConditionVariable wal_replay_cv; + /* + * Used by physical walsenders holding slots specified in + * standby_slot_names to wake up logical walsenders holding + * failover-enabled slots when a walreceiver confirms the receipt of LSN. + */ + ConditionVariable wal_confirm_rcv_cv; + WalSnd walsnds[FLEXIBLE_ARRAY_MEMBER]; } WalSndCtlData; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 47854b5cd4..a9bba11187 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -258,7 +258,9 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); extern bool AllTablesyncsReady(void); -extern void UpdateTwoPhaseState(Oid suboid, char new_state); +extern void UpdateTwoPhaseFailoverStates(Oid suboid, + bool update_twophase, char new_state_twophase, + bool update_failover, char new_state_failover); extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..2f3028cc07 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -162,5 +162,8 @@ extern bool check_wal_consistency_checking(char **newval, void **extra, extern void assign_wal_consistency_checking(const char *newval, void *extra); extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); +extern bool check_standby_slot_names(char **newval, void **extra, + GucSource source); +extern void assign_standby_slot_names(const char *newval, void *extra); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 9d8039684a..3be3ee52fc 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -45,6 +45,7 @@ tests += { 't/037_invalid_database.pl', 't/038_save_logical_slots_shutdown.pl', 't/039_end_of_wal.pl', + 't/050_verify_slot_order.pl', ], }, } diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 5025d65b1b..a3c3ee3a14 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -172,9 +172,10 @@ is($node_primary->slot('otherdb_slot')->{'slot_name'}, undef, 'logical slot was actually dropped with DB'); # Test logical slot advancing and its durability. +# Pass failover=true (last-arg), it should not have any impact on advancing. my $logical_slot = 'logical_slot'; $node_primary->safe_psql('postgres', - "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false);" + "SELECT pg_create_logical_replication_slot('$logical_slot', 'test_decoding', false, false, true);" ); $node_primary->psql( 'postgres', " diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl new file mode 100644 index 0000000000..42e51634c5 --- /dev/null +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -0,0 +1,145 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test primary disallowing specified logical replication slots getting ahead of +# specified physical replication slots. It uses the following set up: +# +# | ----> standby1 (connected via streaming replication) +# | ----> standby2 (connected via streaming replication) +# primary ----- | +# | ----> subscriber1 (connected via logical replication) +# | ----> subscriber2 (connected via logical replication) +# +# Set up is configured in such a way that primary never lets subscriber1 ahead +# of standby1. + +# Create primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Configure primary to disallow specified logical replication slot (lsub1_slot) +# getting ahead of specified physical replication slot (sb1_slot). +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb1_slot' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb1_slot');}); +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb2_slot');}); + +$primary->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +my $backup_name = 'backup'; +$primary->backup($backup_name); + +# Create a standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby1->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb1_slot' +)); +$standby1->start; +$primary->wait_for_replay_catchup($standby1); + +# Create another standby +my $standby2 = PostgreSQL::Test::Cluster->new('standby2'); +$standby2->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); +$standby2->append_conf( + 'postgresql.conf', qq( +primary_slot_name = 'sb2_slot' +)); +$standby2->start; +$primary->wait_for_replay_catchup($standby2); + +# Create publication on primary +my $publisher = $primary; +$publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR TABLE tab_int;"); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; + +# Create a subscriber node, wait for sync to complete +my $subscriber1 = PostgreSQL::Test::Cluster->new('subscriber1'); +$subscriber1->init(allows_streaming => 'logical'); +$subscriber1->start; +$subscriber1->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); + +# Create a subscription with failover = true +$subscriber1->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub1 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub1_slot, failover = true);"); +$subscriber1->wait_for_subscription_sync; + +# Create another subscriber node, wait for sync to complete +my $subscriber2 = PostgreSQL::Test::Cluster->new('subscriber2'); +$subscriber2->init(allows_streaming => 'logical'); +$subscriber2->start; +$subscriber2->safe_psql('postgres', "CREATE TABLE tab_int (a int PRIMARY KEY);"); +$subscriber2->safe_psql('postgres', + "CREATE SUBSCRIPTION mysub2 CONNECTION '$publisher_connstr' " + . "PUBLICATION mypub WITH (slot_name = lsub2_slot);"); +$subscriber2->wait_for_subscription_sync; + +# Stop the standby associated with specified physical replication slot so that +# the logical replication slot won't receive changes until the standby comes +# up. +$standby1->stop; + +# Create some data on primary +my $primary_row_count = 10; +my $primary_insert_time = time(); +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# Wait for the standby that's up and running gets the data from primary +$primary->wait_for_replay_catchup($standby2); +my $result = $standby2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby2 gets data from primary"); + +# Wait for the subscription that's up and running and is not enabled for failover. +# It gets the data from primary without waiting for any standbys. +$publisher->wait_for_catchup('mysub2'); +$result = $subscriber2->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber2 gets data from primary"); + +# The subscription that's up and running and is enabled for failover +# doesn't get the data from primary and keeps waiting for the +# standby specified in standby_slot_names. +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = 0 FROM tab_int;"); +is($result, 't', "subscriber1 doesn't get data from primary until standby1 acknowledges changes"); + +# Start the standby specified in standby_slot_names and wait for it to catch +# up with the primary. +$standby1->start; +$primary->wait_for_replay_catchup($standby1); +$result = $standby1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "standby1 gets data from primary"); + +# Now that the standby specified in standby_slot_names is up and running, +# primary must send the decoded changes to subscription enabled for failover +# While the standby was down, this subscriber didn't receive any data from +# primary i.e. the primary didn't allow it to go ahead of standby. +$publisher->wait_for_catchup('mysub1'); +$result = $subscriber1->safe_psql('postgres', + "SELECT count(*) = $primary_row_count FROM tab_int;"); +is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 1442c43d9c..c9647e86b2 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1473,8 +1473,9 @@ pg_replication_slots| SELECT l.slot_name, l.wal_status, l.safe_wal_size, l.two_phase, - l.conflicting - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting) + l.conflicting, + l.failover + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index b15eddbff3..7c0ef94330 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -116,18 +116,18 @@ CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PU WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | none | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub4 SET (origin = any); \dRs+ regress_testsub4 - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN -------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub4 | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub3; @@ -145,10 +145,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -157,10 +157,10 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname'); ALTER SUBSCRIPTION regress_testsub SET (password_required = false); ALTER SUBSCRIPTION regress_testsub SET (run_as_owner = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | f | t | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (password_required = true); @@ -176,10 +176,10 @@ ERROR: unrecognized subscription parameter: "create_slot" -- ok ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/12345'); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/12345 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/12345 (1 row) -- ok - with lsn = NONE @@ -188,10 +188,10 @@ ALTER SUBSCRIPTION regress_testsub SKIP (lsn = NONE); ALTER SUBSCRIPTION regress_testsub SKIP (lsn = '0/0'); ERROR: invalid WAL location (LSN): 0/0 \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist2 | 0/0 (1 row) BEGIN; @@ -223,10 +223,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ----------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+------------------------------+---------- - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | local | dbname=regress_doesnotexist2 | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +---------------------+---------------------------+---------+---------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+------------------------------+---------- + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | off | d | f | any | t | f | d | local | dbname=regress_doesnotexist2 | 0/0 (1 row) -- rename back to keep the rest simple @@ -255,19 +255,19 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | t | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -279,27 +279,27 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | parallel | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication already exists @@ -314,10 +314,10 @@ ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refr ALTER SUBSCRIPTION regress_testsub ADD PUBLICATION testpub1, testpub2 WITH (refresh = false); ERROR: publication "testpub1" is already in subscription "regress_testsub" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-----------------------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub,testpub1,testpub2} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) -- fail - publication used more than once @@ -332,10 +332,10 @@ ERROR: publication "testpub3" is not in subscription "regress_testsub" -- ok - delete publications ALTER SUBSCRIPTION regress_testsub DROP PUBLICATION testpub1, testpub2 WITH (refresh = false); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) DROP SUBSCRIPTION regress_testsub; @@ -371,10 +371,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) --fail - alter of two_phase option not supported. @@ -383,10 +383,10 @@ ERROR: unrecognized subscription parameter: "two_phase" -- but can alter streaming when two_phase enabled ALTER SUBSCRIPTION regress_testsub SET (streaming = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -396,10 +396,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | on | p | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); @@ -412,18 +412,18 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB WARNING: subscription was created, but is not connected HINT: To initiate replication, you must manually create the replication slot, enable the subscription, and refresh the subscription. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | f | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (disable_on_error = true); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Synchronous commit | Conninfo | Skip LSN ------------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+--------------------+-----------------------------+---------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | off | dbname=regress_doesnotexist | 0/0 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Password required | Run as owner? | Failover | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+-------------------+---------------+----------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | off | d | t | any | t | f | d | off | dbname=regress_doesnotexist | 0/0 (1 row) ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dba3498a13..bc057c74d8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -85,6 +85,7 @@ AlterOwnerStmt AlterPolicyStmt AlterPublicationAction AlterPublicationStmt +AlterReplicationSlotCmd AlterRoleSetStmt AlterRoleStmt AlterSeqStmt @@ -3861,6 +3862,7 @@ varattrib_1b_e varattrib_4b vbits verifier_context +walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn walrcv_create_slot_fn -- 2.34.1 [application/octet-stream] v38-0002-Add-logical-slot-sync-capability-to-the-physical.patch (101.6K, ../../CAJpy0uD6dWUvBgy8MGdugf_Am4pLXTL_vqcwSeHO13v+Mzc9KA@mail.gmail.com/4-v38-0002-Add-logical-slot-sync-capability-to-the-physical.patch) download | inline diff: From 2ee2246ea6a5d22b4e3e55cf47385a24830de887 Mon Sep 17 00:00:00 2001 From: Shveta Malik <[email protected]> Date: Tue, 7 Nov 2023 12:18:59 +0530 Subject: [PATCH v38 2/3] Add logical slot sync capability to the physical standby This patch implements synchronization of logical replication slots from the primary server to the physical standby so that logical replication can be resumed after failover. All the failover logical replication slots on the primary (assuming configurations are appropriate) are automatically created on the physical standbys and are synced periodically. Slot-sync worker on the standby server ping the primary server at regular intervals to get the necessary failover logical slots information and create/update the slots locally. GUC 'enable_syncslot' enables a physical standby to synchronize failover logical replication slots from the primary server. The replication launcher on the physical standby starts slot-sync worker which is then responsible to keep on syncing the logical failover slots from the primary server. The nap time of worker is tuned according to the activity on the primary. The worker starts with nap time of 10ms and if no activity is observed on the primary for some time, then nap time is increased to 10sec. And if activity is observed again, nap time is reduced back to 10ms. The logical slots created by slot-sync worker on physical standbys are not allowed to be dropped or consumed. Any attempt to perform logical decoding on such slots will result in an error. If a logical slot is invalidated on the primary, slot on the standby is also invalidated. If a logical slot on the primary is valid but is invalidated on the standby due to conflict (say required rows removed on the primary), then that slot is dropped and recreated on the standby in next sync-cycle. It is okay to recreate such slots as long as these are not consumable on the standby (which is the case currently). Slots synced on the standby can be identified using 'sync_state' column of pg_replication_slots view. The values are: 'n': none for user slots, 'i': sync initiated for the slot but waiting for the remote slot on the primary server to catch up. 'r': ready for periodic syncs. --- doc/src/sgml/config.sgml | 29 +- doc/src/sgml/system-views.sgml | 23 + src/backend/access/transam/xlogrecovery.c | 11 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/bgworker.c | 5 +- .../libpqwalreceiver/libpqwalreceiver.c | 44 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 3 +- src/backend/replication/logical/launcher.c | 659 +++++++++-- src/backend/replication/logical/logical.c | 22 + src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/slotsync.c | 1018 +++++++++++++++++ src/backend/replication/logical/tablesync.c | 5 +- src/backend/replication/slot.c | 18 +- src/backend/replication/slotfuncs.c | 33 +- src/backend/replication/walsender.c | 2 +- src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/catalog/pg_proc.dat | 10 +- src/include/commands/subscriptioncmds.h | 4 + src/include/nodes/replnodes.h | 1 - src/include/replication/logicallauncher.h | 9 +- src/include/replication/logicalworker.h | 1 + src/include/replication/slot.h | 20 +- src/include/replication/walreceiver.h | 19 + src/include/replication/worker_internal.h | 63 +- src/test/recovery/t/050_verify_slot_order.pl | 127 ++ src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/sysviews.out | 3 +- src/tools/pgindent/typedefs.list | 5 + 32 files changed, 2007 insertions(+), 153 deletions(-) create mode 100644 src/backend/replication/logical/slotsync.c diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e8fbe8ca55..b2da901ad6 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4556,10 +4556,15 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" A password needs to be provided too, if the sender demands password authentication. It can be provided in the <varname>primary_conninfo</varname> string, or in a separate - <filename>~/.pgpass</filename> file on the standby server (use + <filename>~/.pgpass</filename> file on the standby server. (use <literal>replication</literal> as the database name). - Do not specify a database name in the - <varname>primary_conninfo</varname> string. + </para> + <para> + Specify <literal>dbname</literal> in + <varname>primary_conninfo</varname> string to allow synchronization + of slots from the primary server to the standby server. + This will only be used for slot synchronization. It is ignored + for streaming. </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> @@ -4884,6 +4889,24 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class=" </listitem> </varlistentry> + <varlistentry id="guc-enable-syncslot" xreflabel="enable_syncslot"> + <term><varname>enable_syncslot</varname> (<type>boolean</type>) + <indexterm> + <primary><varname>enable_syncslot</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + It enables a physical standby to synchronize logical failover slots + from the primary server so that logical subscribers are not blocked + after failover. + </para> + <para> + It is disabled by default. This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. + </para> + </listitem> + </varlistentry> </variablelist> </sect2> diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 7ea08942c4..dedba3f3ee 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2543,6 +2543,29 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx after failover. Always false for physical slots. </para></entry> </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>sync_state</structfield> <type>char</type> + </para> + <para> + Defines slot synchronization state. This is meaningful on the physical + standby which has enabled slots synchronization. + </para> + <para> + State code: + <literal>n</literal> = none for user created slots, + <literal>i</literal> = sync initiated for the slot but slot is not ready + yet for periodic syncs, + <literal>r</literal> = ready for periodic syncs + </para> + <para> + The primary server will have sync_state as 'n' for all the slots. + But if the standby is promoted to become the new primary server, + sync_state can be seen 'r' as well. On this new primary server, slots + with sync_state as 'r' and 'n' behaves the same. + </para></entry> + </row> </tbody> </tgroup> </table> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c61566666a..8ea6dc799a 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -49,7 +49,9 @@ #include "postmaster/bgwriter.h" #include "postmaster/startup.h" #include "replication/slot.h" +#include "replication/logicallauncher.h" #include "replication/walreceiver.h" +#include "replication/worker_internal.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -1435,6 +1437,15 @@ FinishWalRecovery(void) */ XLogShutdownWalRcv(); + /* + * Shutdown the slot sync workers to prevent potential conflicts between + * user processes and slotsync workers after a promotion. Additionally, + * drop any slots that have initiated but not yet completed the sync + * process. + */ + ShutDownSlotSync(); + slotsync_drop_initiated_slots(); + /* * We are now done reading the xlog from stream. Turn off streaming * recovery to force fetching the files (which would be required at end of diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 798c8d705d..750bebc124 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1003,7 +1003,8 @@ CREATE VIEW pg_replication_slots AS L.safe_wal_size, L.two_phase, L.conflicting, - L.failover + L.failover, + L.sync_state FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 48a9924527..0e039c786f 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -125,11 +125,14 @@ static const struct "ParallelWorkerMain", ParallelWorkerMain }, { - "ApplyLauncherMain", ApplyLauncherMain + "LauncherMain", LauncherMain }, { "ApplyWorkerMain", ApplyWorkerMain }, + { + "ReplSlotSyncWorkerMain", ReplSlotSyncWorkerMain + }, { "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 336c2bec99..5f82d01e6d 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -34,6 +34,7 @@ #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/tuplestore.h" +#include "utils/varlena.h" PG_MODULE_MAGIC; @@ -58,6 +59,7 @@ static void libpqrcv_get_senderinfo(WalReceiverConn *conn, char **sender_host, int *sender_port); static char *libpqrcv_identify_system(WalReceiverConn *conn, TimeLineID *primary_tli); +static char *libpqrcv_get_dbname_from_conninfo(const char *conninfo); static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, @@ -100,6 +102,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { .walrcv_send = libpqrcv_send, .walrcv_create_slot = libpqrcv_create_slot, .walrcv_alter_slot = libpqrcv_alter_slot, + .walrcv_get_dbname_from_conninfo = libpqrcv_get_dbname_from_conninfo, .walrcv_get_backend_pid = libpqrcv_get_backend_pid, .walrcv_exec = libpqrcv_exec, .walrcv_disconnect = libpqrcv_disconnect @@ -413,6 +416,45 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Get database name from the primary server's conninfo. + * + * If dbname is not found in connInfo, return NULL value. + */ +static char * +libpqrcv_get_dbname_from_conninfo(const char *connInfo) +{ + PQconninfoOption *opts; + PQconninfoOption *opt; + char *dbname = NULL; + char *err = NULL; + + opts = PQconninfoParse(connInfo, &err); + if (opts == NULL) + { + /* The error string is malloc'd, so we must free it explicitly */ + char *errcopy = err ? pstrdup(err) : "out of memory"; + + PQfreemem(err); + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid connection string syntax: %s", errcopy))); + } + + for (opt = opts; opt->keyword != NULL; ++opt) + { + /* + * If multiple dbnames are specified, then the last one will be + * returned + */ + if (strcmp(opt->keyword, "dbname") == 0 && opt->val && + opt->val[0] != '\0') + dbname = pstrdup(opt->val); + } + + return dbname; +} + /* * Start streaming WAL data from given streaming options. * @@ -998,7 +1040,7 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, */ static void libpqrcv_alter_slot(WalReceiverConn *conn, const char *slotname, - bool failover) + bool failover) { StringInfoData cmd; PGresult *res; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 2dc25e37bb..ba03eeff1c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -25,6 +25,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + slotsync.o \ snapbuild.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index 9b37736f8e..192c9e1860 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -922,7 +922,8 @@ ParallelApplyWorkerMain(Datum main_arg) before_shmem_exit(pa_shutdown, PointerGetDatum(seg)); SpinLockAcquire(&MyParallelShared->mutex); - MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_generation = + MyLogicalRepWorker->hdr.generation; MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 501910b445..acafb54276 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -8,20 +8,27 @@ * src/backend/replication/logical/launcher.c * * NOTES - * This module contains the logical replication worker launcher which - * uses the background worker infrastructure to start the logical - * replication workers for every enabled subscription. + * This module contains the replication worker launcher which + * uses the background worker infrastructure to: + * a) start the logical replication workers for every enabled subscription + * when not in standby_mode. + * b) start the slot sync worker for logical failover slots synchronization + * from the primary server when in standby_mode. * *------------------------------------------------------------------------- */ #include "postgres.h" +#include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/tableam.h" #include "access/xact.h" +#include "access/xlogrecovery.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_database.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "funcapi.h" @@ -44,6 +51,7 @@ #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/ps_status.h" @@ -57,6 +65,9 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; int max_parallel_apply_workers_per_subscription = 2; +bool enable_syncslot = false; + +SlotSyncWorkerInfo *SlotSyncWorker = NULL; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -70,6 +81,7 @@ typedef struct LogicalRepCtxStruct dshash_table_handle last_start_dsh; /* Background workers. */ + SlotSyncWorkerInfo ss_worker; /* slot sync worker */ LogicalRepWorker workers[FLEXIBLE_ARRAY_MEMBER]; } LogicalRepCtxStruct; @@ -102,6 +114,7 @@ static void logicalrep_launcher_onexit(int code, Datum arg); static void logicalrep_worker_onexit(int code, Datum arg); static void logicalrep_worker_detach(void); static void logicalrep_worker_cleanup(LogicalRepWorker *worker); +static void slotsync_worker_cleanup(SlotSyncWorkerInfo *worker); static int logicalrep_pa_worker_count(Oid subid); static void logicalrep_launcher_attach_dshmem(void); static void ApplyLauncherSetWorkerStartTime(Oid subid, TimestampTz start_time); @@ -178,6 +191,8 @@ get_subscription_list(void) } /* + * This is common code for logical workers and slot sync worker. + * * Wait for a background worker to start up and attach to the shmem context. * * This is only needed for cleaning up the shared memory in case the worker @@ -186,12 +201,14 @@ get_subscription_list(void) * Returns whether the attach was successful. */ static bool -WaitForReplicationWorkerAttach(LogicalRepWorker *worker, +WaitForReplicationWorkerAttach(LogicalWorkerHeader *worker, uint16 generation, - BackgroundWorkerHandle *handle) + BackgroundWorkerHandle *handle, + LWLock *lock) { BgwHandleStatus status; int rc; + bool is_slotsync_worker = (lock == SlotSyncWorkerLock) ? true : false; for (;;) { @@ -199,27 +216,32 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, CHECK_FOR_INTERRUPTS(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); return worker->in_use; } - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Check if worker has died before attaching, and clean up after it. */ status = GetBackgroundWorkerPid(handle, &pid); if (status == BGWH_STOPPED) { - LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(lock, LW_EXCLUSIVE); /* Ensure that this was indeed the worker we waited for. */ if (generation == worker->generation) - logicalrep_worker_cleanup(worker); - LWLockRelease(LogicalRepWorkerLock); + { + if (is_slotsync_worker) + slotsync_worker_cleanup((SlotSyncWorkerInfo *) worker); + else + logicalrep_worker_cleanup((LogicalRepWorker *) worker); + } + LWLockRelease(lock); return false; } @@ -262,8 +284,8 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) if (isParallelApplyWorker(w)) continue; - if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && w->relid == relid && + (!only_running || w->hdr.proc)) { res = w; break; @@ -290,7 +312,8 @@ logicalrep_workers_find(Oid subid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->in_use && w->subid == subid && (!only_running || w->proc)) + if (w->hdr.in_use && w->subid == subid && + (!only_running || w->hdr.proc)) res = lappend(res, w); } @@ -351,7 +374,7 @@ retry: { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (!w->in_use) + if (!w->hdr.in_use) { worker = w; slot = i; @@ -380,7 +403,7 @@ retry: * If the worker was marked in use but didn't manage to attach in * time, clean it up. */ - if (w->in_use && !w->proc && + if (w->hdr.in_use && !w->hdr.proc && TimestampDifferenceExceeds(w->launch_time, now, wal_receiver_timeout)) { @@ -438,9 +461,9 @@ retry: /* Prepare the worker slot. */ worker->type = wtype; worker->launch_time = now; - worker->in_use = true; - worker->generation++; - worker->proc = NULL; + worker->hdr.in_use = true; + worker->hdr.generation++; + worker->hdr.proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; @@ -457,7 +480,7 @@ retry: TIMESTAMP_NOBEGIN(worker->reply_time); /* Before releasing lock, remember generation for future identification. */ - generation = worker->generation; + generation = worker->hdr.generation; LWLockRelease(LogicalRepWorkerLock); @@ -510,7 +533,7 @@ retry: { /* Failed to start worker, so clean up the worker slot. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); - Assert(generation == worker->generation); + Assert(generation == worker->hdr.generation); logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); @@ -522,19 +545,23 @@ retry: } /* Now wait until it attaches. */ - return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + generation, + bgw_handle, + LogicalRepWorkerLock); } /* * Internal function to stop the worker and wait until it detaches from the - * slot. + * slot. It is used for both logical workers and slot sync worker. */ static void -logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) +logicalrep_worker_stop_internal(LogicalWorkerHeader *worker, int signo, + LWLock *lock) { uint16 generation; - Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); + Assert(LWLockHeldByMeInMode(lock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -550,7 +577,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) { int rc; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -564,7 +591,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) } /* Recheck worker status. */ - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); /* * Check whether the worker slot is no longer used, which would mean @@ -591,7 +618,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) if (!worker->proc || worker->generation != generation) break; - LWLockRelease(LogicalRepWorkerLock); + LWLockRelease(lock); /* Wait a bit --- we don't expect to have to wait long. */ rc = WaitLatch(MyLatch, @@ -604,7 +631,7 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) CHECK_FOR_INTERRUPTS(); } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + LWLockAcquire(lock, LW_SHARED); } } @@ -623,7 +650,9 @@ logicalrep_worker_stop(Oid subid, Oid relid) if (worker) { Assert(!isParallelApplyWorker(worker)); - logicalrep_worker_stop_internal(worker, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -669,8 +698,10 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) /* * Only stop the worker if the generation matches and the worker is alive. */ - if (worker->generation == generation && worker->proc) - logicalrep_worker_stop_internal(worker, SIGINT); + if (worker->hdr.generation == generation && worker->hdr.proc) + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + LogicalRepWorkerLock); LWLockRelease(LogicalRepWorkerLock); } @@ -696,14 +727,14 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) /* * Wake up (using latch) the specified logical replication worker. * - * Caller must hold lock, else worker->proc could change under us. + * Caller must hold lock, else worker->hdr.proc could change under us. */ void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) { Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - SetLatch(&worker->proc->procLatch); + SetLatch(&worker->hdr.proc->procLatch); } /* @@ -718,7 +749,7 @@ logicalrep_worker_attach(int slot) Assert(slot >= 0 && slot < max_logical_replication_workers); MyLogicalRepWorker = &LogicalRepCtx->workers[slot]; - if (!MyLogicalRepWorker->in_use) + if (!MyLogicalRepWorker->hdr.in_use) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -727,7 +758,7 @@ logicalrep_worker_attach(int slot) slot))); } - if (MyLogicalRepWorker->proc) + if (MyLogicalRepWorker->hdr.proc) { LWLockRelease(LogicalRepWorkerLock); ereport(ERROR, @@ -736,7 +767,7 @@ logicalrep_worker_attach(int slot) "another worker, cannot attach", slot))); } - MyLogicalRepWorker->proc = MyProc; + MyLogicalRepWorker->hdr.proc = MyProc; before_shmem_exit(logicalrep_worker_onexit, (Datum) 0); LWLockRelease(LogicalRepWorkerLock); @@ -771,7 +802,9 @@ logicalrep_worker_detach(void) LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); if (isParallelApplyWorker(w)) - logicalrep_worker_stop_internal(w, SIGTERM); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) w, + SIGTERM, + LogicalRepWorkerLock); } LWLockRelease(LogicalRepWorkerLock); @@ -794,10 +827,10 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_EXCLUSIVE)); worker->type = WORKERTYPE_UNKNOWN; - worker->in_use = false; - worker->proc = NULL; - worker->dbid = InvalidOid; + worker->hdr.in_use = false; + worker->hdr.proc = NULL; worker->userid = InvalidOid; + worker->dbid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; worker->leader_pid = InvalidPid; @@ -931,9 +964,18 @@ ApplyLauncherRegister(void) memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* + * The launcher now takes care of launching both logical apply workers and + * logical slot sync worker. Thus to cater to the requirements of both, + * start it as soon as a consistent state is reached. This will help + * slot sync worker to start timely on a physical standby while on a + * non-standby server, it holds same meaning as that of + * BgWorkerStart_RecoveryFinished. + */ + bgw.bgw_start_time = BgWorkerStart_ConsistentState; snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, @@ -1115,13 +1157,447 @@ ApplyLauncherWakeup(void) } /* - * Main loop for the apply launcher process. + * Clean up slot sync worker info. + */ +static void +slotsync_worker_cleanup(SlotSyncWorkerInfo *worker) +{ + Assert(LWLockHeldByMeInMode(SlotSyncWorkerLock, LW_EXCLUSIVE)); + + worker->hdr.in_use = false; + worker->hdr.proc = NULL; + worker->last_update_time = 0; +} + +/* + * Attach slot sync worker to SlotSyncWorkerInfo assigned by the launcher. */ void -ApplyLauncherMain(Datum main_arg) +slotsync_worker_attach() { - ereport(DEBUG1, - (errmsg_internal("logical replication launcher started"))); + /* Block concurrent access. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + if (!SlotSyncWorker->hdr.in_use) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot sync worker not initialized, cannot attach"))); + } + + if (SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("replication slot sync worker is already running, cannot attach"))); + } + + SlotSyncWorker->hdr.proc = MyProc; + + before_shmem_exit(slotsync_worker_detach, (Datum) 0); + + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Detach the slot sync worker (cleans up the worker info). + */ +void +slotsync_worker_detach(int code, Datum arg) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(SlotSyncWorker); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Start slot sync background worker. + * + * Returns true on success, false on failure. + */ +static bool +slotsync_worker_launch() +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + SlotSyncWorkerInfo *worker; + bool attach; + + /* The shared memory must only be modified under lock. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + + SlotSyncWorker = &LogicalRepCtx->ss_worker; + + worker = SlotSyncWorker; + + /* Prepare the new worker. */ + worker->hdr.in_use = true; + + /* + * 'proc' will be assigned in ReplSlotSyncWorkerMain when the worker + * attaches to SlotSyncWorkerInfo. + */ + worker->hdr.proc = NULL; + + LWLockRelease(SlotSyncWorkerLock); + + /* Register the new dynamic worker. */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ReplSlotSyncWorkerMain"); + + snprintf(bgw.bgw_name, BGW_MAXLEN, "replication slot sync worker"); + + snprintf(bgw.bgw_type, BGW_MAXLEN, "slot sync worker"); + + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to start worker, so clean up the worker slot. */ + LWLockAcquire(SlotSyncWorkerLock, LW_EXCLUSIVE); + slotsync_worker_cleanup(worker); + LWLockRelease(SlotSyncWorkerLock); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase %s.", + "max_worker_processes"))); + return false; + } + + /* Now wait until it attaches. */ + attach = WaitForReplicationWorkerAttach((LogicalWorkerHeader *) worker, + worker->hdr.generation, + bgw_handle, + SlotSyncWorkerLock); + + if (!attach) + ereport(WARNING, + (errmsg("replication slot sync worker failed to attach"))); + + return attach; +} + +/* + * Internal function to stop the slot sync worker and cleanup afterwards. + */ +static void +slotsync_worker_stop_internal(SlotSyncWorkerInfo *worker) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + ereport(LOG, + (errmsg("stopping replication slot sync worker with pid: %d", + worker->hdr.proc->pid))); + logicalrep_worker_stop_internal((LogicalWorkerHeader *) worker, + SIGINT, + SlotSyncWorkerLock); + LWLockRelease(SlotSyncWorkerLock); +} + +/* + * Validate if db with given dbname exists. + * + * Can't use existing functions like 'get_database_oid' from dbcommands.c for + * validity purpose as they need db connection. + */ +static bool +validate_dbname(const char *dbname) +{ + HeapTuple tuple; + Relation relation; + SysScanDesc scan; + ScanKeyData key[1]; + bool valid; + + /* Start a transaction so we can access pg_database */ + StartTransactionCommand(); + + /* Form a scan key */ + ScanKeyInit(&key[0], Anum_pg_database_datname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(dbname)); + + /* No db connection, force heap scan */ + relation = table_open(DatabaseRelationId, AccessShareLock); + scan = systable_beginscan(relation, DatabaseNameIndexId, false, + NULL, 1, key); + + tuple = systable_getnext(scan); + + if (HeapTupleIsValid(tuple)) + valid = true; + else + valid = false; + + /* all done */ + systable_endscan(scan); + table_close(relation, AccessShareLock); + + CommitTransactionCommand(); + return valid; +} + +/* + * Checks if GUC are set appropriately before starting slot sync worker + */ +static bool +slotsync_checks(long *wait_time, bool *retry) +{ + char *dbname; + + *retry = false; + + if (!enable_syncslot) + return false; + + /* + * Since the above GUC is set, check that other GUC settings + * (primary_slot_name, hot_standby_feedback, primary_conninfo, wal_level) + * are compatible with slot synchronization. If not, issue warnings. + */ + + /* + * A physical replication slot(primary_slot_name) is required on the + * primary to ensure that the rows needed by the standby are not removed + * after restarting, so that the synchronized slot on the standby will not + * be invalidated. + */ + if (!WalRcv || WalRcv->slotname[0] == '\0') + { + ereport(WARNING, + errmsg("skipping slots synchronization as primary_slot_name is not set")); + + /* + * It's possible that the Walreceiver has not been started yet, adjust + * the wait_time to retry sooner in the next synchronization cycle. + */ + *wait_time = wal_retrieve_retry_interval; + + /* + * Tell caller to retry the connection for the case where + * primary_slot_name is set but Walreceiver is not yet started. + */ + if (PrimarySlotName && strcmp(PrimarySlotName, "") != 0) + *retry = true; + + return false; + } + + /* + * Hot_standby_feedback must be enabled to cooperate with the physical + * replication slot, which allows informing the primary about the xmin and + * catalog_xmin values on the standby. + */ + if (!hot_standby_feedback) + { + ereport(WARNING, + errmsg("skipping slots synchronization as hot_standby_feedback is off")); + return false; + } + + /* + * Logical decoding requires wal_level >= logical and we currently only + * synchronize logical slots. + */ + if (wal_level < WAL_LEVEL_LOGICAL) + { + ereport(WARNING, + errmsg("skipping slots synchronisation as it requires wal_level >= logical")); + return false; + } + + /* + * The slot sync worker needs a database connection for walrcv_exec to + * work. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname is not specified in primary_conninfo")); + return false; + } + + if (!validate_dbname(dbname)) + { + ereport(WARNING, + errmsg("skipping slots synchronization as dbname specified in primary_conninfo is not a valid one")); + return false; + } + + return true; +} + +/* + * Shut down the slot sync worker. + */ +void +ShutDownSlotSync(void) +{ + if (LogicalRepCtx->ss_worker.hdr.in_use) + slotsync_worker_stop_internal(&LogicalRepCtx->ss_worker); +} + +/* + * Re-read the config file. + * + * If one of the slot sync options has changed, stop the slot sync worker + * and set ss_recheck flag to enable the caller to recheck slot sync GUCs + * before restarting the worker + */ +static void +LauncherRereadConfig(bool *ss_recheck) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + char *slotname = pstrdup(PrimarySlotName); + bool syncslot = enable_syncslot; + bool standbyfeedback = hot_standby_feedback; + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* + * If any of the related GUCs changed, stop the slot sync worker. The + * worker will be relaunched in next sync-cycle using the new GUCs. + */ + if ((strcmp(conninfo, PrimaryConnInfo) != 0) || + (strcmp(slotname, PrimarySlotName) != 0) || + (syncslot != enable_syncslot) || + (standbyfeedback != hot_standby_feedback)) + { + ShutDownSlotSync(); + + /* Retry slot sync with new GUCs */ + *ss_recheck = true; + } + + pfree(conninfo); + pfree(slotname); +} + +/* + * Launch slot sync background worker. + */ +static void +LaunchSlotSyncWorker(long *wait_time) +{ + LWLockAcquire(SlotSyncWorkerLock, LW_SHARED); + + /* The worker is running already */ + if (SlotSyncWorker && SlotSyncWorker->hdr.in_use && + SlotSyncWorker->hdr.proc) + { + LWLockRelease(SlotSyncWorkerLock); + return; + } + + LWLockRelease(SlotSyncWorkerLock); + + /* + * If launch failed, adjust the wait_time to retry in the next sync-cycle + * sooner. + */ + if (!slotsync_worker_launch()) + { + *wait_time = Min(*wait_time, wal_retrieve_retry_interval); + } +} + +/* + * Launch logical replication apply workers for enabled subscriptions. + */ +static void +LaunchSubscriptionApplyWorker(long *wait_time) +{ + List *sublist; + ListCell *lc; + MemoryContext subctx; + MemoryContext oldctx; + + /* Use temporary context to avoid leaking memory across cycles. */ + subctx = AllocSetContextCreate(TopMemoryContext, + "Logical Replication Launcher sublist", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + /* Start any missing workers for enabled subscriptions. */ + sublist = get_subscription_list(); + foreach(lc, sublist) + { + Subscription *sub = (Subscription *) lfirst(lc); + LogicalRepWorker *w; + TimestampTz last_start; + TimestampTz now; + long elapsed; + + if (!sub->enabled) + continue; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); + LWLockRelease(LogicalRepWorkerLock); + + if (w != NULL) + continue; /* worker is running already */ + + /* + * If the worker is eligible to start now, launch it. Otherwise, + * adjust wait_time so that we'll wake up as soon as it can be + * started. + * + * Each subscription's apply worker can only be restarted once per + * wal_retrieve_retry_interval, so that errors do not cause us to + * repeatedly restart the worker as fast as possible. In cases where + * a restart is expected (e.g., subscription parameter changes), + * another process should remove the last-start entry for the + * subscription so that the worker can be restarted without waiting + * for wal_retrieve_retry_interval to elapse. + */ + last_start = ApplyLauncherGetWorkerStartTime(sub->oid); + now = GetCurrentTimestamp(); + if (last_start == 0 || + (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= + wal_retrieve_retry_interval) + { + ApplyLauncherSetWorkerStartTime(sub->oid, now); + logicalrep_worker_launch(WORKERTYPE_APPLY, + sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid, + DSM_HANDLE_INVALID); + } + else + { + *wait_time = Min(*wait_time, + wal_retrieve_retry_interval - elapsed); + } + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); +} + +/* + * Main loop for the launcher process. + */ +void +LauncherMain(Datum main_arg) +{ + bool start_slotsync = false; + bool recheck_slotsync = true; + + elog(DEBUG1, "logical replication launcher started"); before_shmem_exit(logicalrep_launcher_onexit, (Datum) 0); @@ -1139,79 +1615,32 @@ ApplyLauncherMain(Datum main_arg) */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); + load_file("libpqwalreceiver", false); + /* Enter main loop */ for (;;) { int rc; - List *sublist; - ListCell *lc; - MemoryContext subctx; - MemoryContext oldctx; long wait_time = DEFAULT_NAPTIME_PER_CYCLE; CHECK_FOR_INTERRUPTS(); - /* Use temporary context to avoid leaking memory across cycles. */ - subctx = AllocSetContextCreate(TopMemoryContext, - "Logical Replication Launcher sublist", - ALLOCSET_DEFAULT_SIZES); - oldctx = MemoryContextSwitchTo(subctx); - - /* Start any missing workers for enabled subscriptions. */ - sublist = get_subscription_list(); - foreach(lc, sublist) + /* + * If it is Hot standby, then try to launch slot sync worker else + * launch apply workers. + */ + if (RecoveryInProgress() && !PromoteIsTriggered()) { - Subscription *sub = (Subscription *) lfirst(lc); - LogicalRepWorker *w; - TimestampTz last_start; - TimestampTz now; - long elapsed; - - if (!sub->enabled) - continue; - - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); - LWLockRelease(LogicalRepWorkerLock); + /* Make validation checks first */ + if (recheck_slotsync) + start_slotsync = slotsync_checks(&wait_time, &recheck_slotsync); - if (w != NULL) - continue; /* worker is running already */ - - /* - * If the worker is eligible to start now, launch it. Otherwise, - * adjust wait_time so that we'll wake up as soon as it can be - * started. - * - * Each subscription's apply worker can only be restarted once per - * wal_retrieve_retry_interval, so that errors do not cause us to - * repeatedly restart the worker as fast as possible. In cases - * where a restart is expected (e.g., subscription parameter - * changes), another process should remove the last-start entry - * for the subscription so that the worker can be restarted - * without waiting for wal_retrieve_retry_interval to elapse. - */ - last_start = ApplyLauncherGetWorkerStartTime(sub->oid); - now = GetCurrentTimestamp(); - if (last_start == 0 || - (elapsed = TimestampDifferenceMilliseconds(last_start, now)) >= wal_retrieve_retry_interval) - { - ApplyLauncherSetWorkerStartTime(sub->oid, now); - logicalrep_worker_launch(WORKERTYPE_APPLY, - sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid, - DSM_HANDLE_INVALID); - } - else - { - wait_time = Min(wait_time, - wal_retrieve_retry_interval - elapsed); - } + /* Start slot sync workers if checks passed */ + if (start_slotsync) + LaunchSlotSyncWorker(&wait_time); } - - /* Switch back to original memory context. */ - MemoryContextSwitchTo(oldctx); - /* Clean the temporary memory. */ - MemoryContextDelete(subctx); + else + LaunchSubscriptionApplyWorker(&wait_time); /* Wait for more work. */ rc = WaitLatch(MyLatch, @@ -1226,10 +1655,7 @@ ApplyLauncherMain(Datum main_arg) } if (ConfigReloadPending) - { - ConfigReloadPending = false; - ProcessConfigFile(PGC_SIGHUP); - } + LauncherRereadConfig(&recheck_slotsync); } /* Not reachable */ @@ -1260,7 +1686,8 @@ GetLeaderApplyWorkerPid(pid_t pid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isParallelApplyWorker(w) && w->proc && pid == w->proc->pid) + if (isParallelApplyWorker(w) && w->hdr.proc && + pid == w->hdr.proc->pid) { leader_pid = w->leader_pid; break; @@ -1298,13 +1725,13 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) memcpy(&worker, &LogicalRepCtx->workers[i], sizeof(LogicalRepWorker)); - if (!worker.proc || !IsBackendPid(worker.proc->pid)) + if (!worker.hdr.proc || !IsBackendPid(worker.hdr.proc->pid)) continue; if (OidIsValid(subid) && worker.subid != subid) continue; - worker_pid = worker.proc->pid; + worker_pid = worker.hdr.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); if (isTablesyncWorker(&worker)) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8288da5277..5bef0138b4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -524,6 +524,28 @@ CreateDecodingContext(XLogRecPtr start_lsn, errmsg("replication slot \"%s\" was not created in this database", NameStr(slot->data.name)))); + /* + * Slots in state SYNCSLOT_STATE_INITIATED should have been dropped on + * promotion. + */ + if (!RecoveryInProgress() && slot->data.sync_state == SYNCSLOT_STATE_INITIATED) + elog(ERROR, "replication slot \"%s\" was not synced completely from the primary server", + NameStr(slot->data.name)); + + /* + * Do not allow consumption of a "synchronized" slot until the standby + * gets promoted. Also do not allow consumption of slots with sync_state + * as SYNCSLOT_STATE_INITIATED as they are not synced completely to be + * used. + */ + if (RecoveryInProgress() && slot->data.sync_state != SYNCSLOT_STATE_NONE) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot use replication slot \"%s\" for logical decoding", + NameStr(slot->data.name)), + errdetail("This slot is being synced from the primary server."), + errhint("Specify another replication slot."))); + /* * Check if slot has been invalidated due to max_slot_wal_keep_size. Avoid * "cannot get changes" wording in this errmsg because that'd be diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index d48cd4c590..9e52ec421f 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -11,6 +11,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'slotsync.c', 'snapbuild.c', 'tablesync.c', 'worker.c', diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c new file mode 100644 index 0000000000..9aa3562500 --- /dev/null +++ b/src/backend/replication/logical/slotsync.c @@ -0,0 +1,1018 @@ +/*------------------------------------------------------------------------- + * slotsync.c + * PostgreSQL worker for synchronizing slots to a standby server from the + * primary server. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/slotsync.c + * + * This file contains the code for slot sync worker on a physical standby + * to fetch logical failover slots information from the primary server, + * create the slots on the standby and synchronize them periodically. + * + * It also takes care of dropping the slots which were created by it and are + * currently not needed to be synchronized. + * + * It takes a nap of WORKER_DEFAULT_NAPTIME_MS before every next + * synchronization. If there is no activity observed on the primary server for + * some time, the nap time is increased to WORKER_INACTIVITY_NAPTIME_MS, but if + * any activity is observed, the nap time reverts to the default value. + *--------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlogrecovery.h" +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "replication/logical.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/guc_hooks.h" +#include "utils/pg_lsn.h" +#include "utils/varlena.h" + +/* + * Structure to hold information fetched from the primary server about a logical + * replication slot. + */ +typedef struct RemoteSlot +{ + char *name; + char *plugin; + char *database; + bool two_phase; + bool failover; + XLogRecPtr restart_lsn; + XLogRecPtr confirmed_lsn; + TransactionId catalog_xmin; + + /* RS_INVAL_NONE if valid, or the reason of invalidation */ + ReplicationSlotInvalidationCause invalidated; +} RemoteSlot; + +/* Worker's nap time in case of regular activity on the primary server */ +#define WORKER_DEFAULT_NAPTIME_MS 10L /* 10 ms */ + +/* Worker's nap time in case of no-activity on the primary server */ +#define WORKER_INACTIVITY_NAPTIME_MS 10000L /* 10 sec */ + +/* + * Inactivity Threshold in ms before increasing nap time of worker. + * + * If the lsn of slot being monitored did not change for this threshold time, + * then increase nap time of current worker from WORKER_DEFAULT_NAPTIME_MS to + * WORKER_INACTIVITY_NAPTIME_MS. + */ +#define WORKER_INACTIVITY_THRESHOLD_MS 10000L /* 10 sec */ + +/* + * Number of attempts for wait_for_primary_slot_catchup() after + * which it aborts the wait and the slot sync worker then moves + * to the next slot creation/sync. + */ +#define WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS 5 + +/* + * Wait for remote slot to pass locally reserved position. + * + * Ping and wait for the primary server for + * WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS during a slot creation, if it still + * does not catch up, abort the wait. The ones for which wait is aborted will + * attempt the wait and sync in the next sync-cycle. + * + * *persist will be set to false if the slot has disappeared or was invalidated + * on the primary; otherwise, it will be set to true. + */ +static bool +wait_for_primary_slot_catchup(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *persist) +{ +#define WAIT_OUTPUT_COLUMN_COUNT 4 + StringInfoData cmd; + int wait_count = 0; + + if (persist) + *persist = true; + + ereport(LOG, + errmsg("waiting for remote slot \"%s\" LSN (%X/%X) and catalog xmin" + " (%u) to pass local slot LSN (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->restart_lsn), + remote_slot->catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT conflicting, restart_lsn, confirmed_flush_lsn," + " catalog_xmin FROM pg_catalog.pg_replication_slots" + " WHERE slot_name = %s", + quote_literal_cstr(remote_slot->name)); + + for (;;) + { + XLogRecPtr new_invalidated; + XLogRecPtr new_restart_lsn; + XLogRecPtr new_confirmed_lsn; + TransactionId new_catalog_xmin; + WalRcvExecResult *res; + TupleTableSlot *slot; + int rc; + bool isnull; + Oid slotRow[WAIT_OUTPUT_COLUMN_COUNT] = {BOOLOID, LSNOID, LSNOID, + XIDOID}; + + CHECK_FOR_INTERRUPTS(); + + Assert(RecoveryInProgress()); + + res = walrcv_exec(wrconn, cmd.data, WAIT_OUTPUT_COLUMN_COUNT, slotRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch slot info for slot \"%s\" from the" + " primary server: %s", + remote_slot->name, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + ereport(WARNING, + (errmsg("slot \"%s\" disappeared from the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + walrcv_clear_result(res); + + /* + * The slot being created will be dropped when it is released (see + * ReplicationSlotRelease). + */ + if (persist) + *persist = false; + + return false; + } + + + /* + * It is possible to get null values for LSN and Xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + new_invalidated = DatumGetBool(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + new_restart_lsn = DatumGetLSN(slot_getattr(slot, 2, &isnull)); + if (new_invalidated || isnull) + { + ereport(WARNING, + (errmsg("slot \"%s\" invalidated on the primary server," + " slot creation aborted", remote_slot->name))); + pfree(cmd.data); + ExecClearTuple(slot); + walrcv_clear_result(res); + + /* + * The slot being created will be dropped when it is released (see + * ReplicationSlotRelease). + */ + if (persist) + *persist = false; + + return false; + } + + /* + * Once we got valid restart_lsn, then confirmed_lsn and catalog_xmin + * are expected to be valid/non-null. + */ + new_confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + Assert(!isnull); + + new_catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 4, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + walrcv_clear_result(res); + + if (new_restart_lsn >= MyReplicationSlot->data.restart_lsn && + TransactionIdFollowsOrEquals(new_catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + /* Update new values in remote_slot */ + remote_slot->restart_lsn = new_restart_lsn; + remote_slot->confirmed_lsn = new_confirmed_lsn; + remote_slot->catalog_xmin = new_catalog_xmin; + + ereport(LOG, + errmsg("wait over for remote slot \"%s\" as its LSN (%X/%X)" + " and catalog xmin (%u) has now passed local slot LSN" + " (%X/%X) and catalog xmin (%u)", + remote_slot->name, + LSN_FORMAT_ARGS(new_restart_lsn), + new_catalog_xmin, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + MyReplicationSlot->data.catalog_xmin)); + pfree(cmd.data); + + return true; + } + + if (++wait_count >= WORKER_PRIMARY_CATCHUP_WAIT_ATTEMPTS) + { + ereport(LOG, + errmsg("aborting the wait for remote slot \"%s\" and moving" + " to the next slot, will attempt creating it again", + remote_slot->name)); + pfree(cmd.data); + + return false; + } + + /* + * XXX: Is waiting for 2 seconds before retrying enough or more or + * less? + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 2000L, + WAIT_EVENT_REPL_SLOTSYNC_PRIMARY_CATCHUP); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Update local slot metadata as per remote_slot's positions + */ +static void +local_slot_update(RemoteSlot *remote_slot) +{ + Assert(MyReplicationSlot->data.invalidated == RS_INVAL_NONE); + + LogicalConfirmReceivedLocation(remote_slot->confirmed_lsn); + LogicalIncreaseXminForSlot(remote_slot->confirmed_lsn, + remote_slot->catalog_xmin); + LogicalIncreaseRestartDecodingForSlot(remote_slot->confirmed_lsn, + remote_slot->restart_lsn); + + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotMarkDirty(); +} + +/* + * Drop the slots for which sync is initiated but not yet completed + * i.e. they are still waiting for the primary server to catch up. + */ +void +slotsync_drop_initiated_slots(void) +{ + List *slots = NIL; + ListCell *lc; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + if (s->in_use && s->data.sync_state == SYNCSLOT_STATE_INITIATED) + slots = lappend(slots, s); + } + + LWLockRelease(ReplicationSlotControlLock); + + foreach(lc, slots) + { + ReplicationSlot *s = (ReplicationSlot *) lfirst(lc); + + ReplicationSlotDrop(NameStr(s->data.name), true, false); + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d as it " + "was not sync-ready", NameStr(s->data.name), + s->data.database))); + } + + list_free(slots); +} + +/* + * Get list of local logical slot names which are synchronized from + * the primary server. + */ +static List * +get_local_synced_slot_names(void) +{ + List *localSyncedSlots = NIL; + + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + + for (int i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i]; + + /* Check if it is logical synchronized slot */ + if (s->in_use && SlotIsLogical(s) && + (s->data.sync_state != SYNCSLOT_STATE_NONE)) + { + localSyncedSlots = lappend(localSyncedSlots, s); + } + } + + LWLockRelease(ReplicationSlotControlLock); + + return localSyncedSlots; +} + +/* + * Helper function to check if local_slot is present in remote_slots list. + * + * It also checks if logical slot is locally invalidated i.e. invalidated on + * the standby but valid on the primary server. If found so, it sets + * locally_invalidated to true. + */ +static bool +validate_sync_slot(ReplicationSlot *local_slot, List *remote_slots, + bool *locally_invalidated) +{ + ListCell *cell; + + foreach(cell, remote_slots) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0) + { + /* + * If remote slot is not invalidated but local slot is marked as + * invalidated, then set the bool. + */ + *locally_invalidated = + (remote_slot->invalidated == RS_INVAL_NONE) && + (local_slot->data.invalidated != RS_INVAL_NONE); + + return true; + } + } + + return false; +} + +/* + * Drop obsolete slots + * + * Drop the slots that no longer need to be synced i.e. these either do not + * exist on the primary or are no longer enabled for failover. + * + * Also drop the slots that are valid on the primary that got invalidated + * on the standby due to conflict (say required rows removed on the primary). + * The assumption is, that these will get recreated in next sync-cycle and + * it is okay to drop and recreate such slots as long as these are not + * consumable on the standby (which is the case currently). + */ +static void +drop_obsolete_slots(List *remote_slot_list) +{ + List *local_slot_list = NIL; + ListCell *lc_slot; + + /* + * Get the list of local 'synced' slot so that those not on remote could + * be dropped. + */ + local_slot_list = get_local_synced_slot_names(); + + foreach(lc_slot, local_slot_list) + { + ReplicationSlot *local_slot = (ReplicationSlot *) lfirst(lc_slot); + bool local_exists = false; + bool locally_invalidated = false; + + local_exists = validate_sync_slot(local_slot, remote_slot_list, + &locally_invalidated); + + /* + * Drop the local slot either if it is not in the remote slots list or + * is invalidated while remote slot is still valid. + */ + if (!local_exists || locally_invalidated) + { + ReplicationSlotDrop(NameStr(local_slot->data.name), true, false); + + ereport(LOG, + (errmsg("dropped replication slot \"%s\" of dbid %d", + NameStr(local_slot->data.name), + local_slot->data.database))); + } + } +} + +/* + * Constructs the query in order to get failover logical slots + * information from the primary server. + */ +static void +construct_slot_query(StringInfo s) +{ + /* + * Fetch slots with failover enabled. + * + * If we are on cascading standby, we should fetch only those slots from + * the first standby which have sync_state as either 'n' or 'r'. Slots + * with sync_state as 'i' are not sync ready yet. And when we are on the + * first standby, the primary server is supposed to have slots with + * sync_state as 'n' only (or it may have all 'n', 'r' and 'i' if standby + * is promoted as primary). Thus in all the cases, filter sync_state !='i' + * is appropriate one. + */ + appendStringInfo(s, + "SELECT slot_name, plugin, confirmed_flush_lsn," + " restart_lsn, catalog_xmin, two_phase, failover," + " database, pg_get_slot_invalidation_cause(slot_name)" + " FROM pg_catalog.pg_replication_slots" + " WHERE failover and sync_state != 'i'"); +} + +/* + * Synchronize single slot to given position. + * + * This creates a new slot if there is no existing one and updates the + * metadata of the slot as per the data received from the primary server. + * + * The 'sync_state' in slot.data is set to SYNCSLOT_STATE_INITIATED + * immediately after creation. It stays in same state until the + * initialization is complete. The initialization is considered to + * be completed once the remote_slot catches up with locally reserved + * position and local slot is updated. The sync_state is then changed + * to SYNCSLOT_STATE_READY. + */ +static void +synchronize_one_slot(WalReceiverConn *wrconn, RemoteSlot *remote_slot, + bool *slot_updated) +{ + ReplicationSlot *s; + char sync_state = 0; + + /* + * Make sure that concerned WAL is received before syncing slot to target + * lsn received from the primary server. + * + * This check should never pass as on the primary server, we have waited + * for the standby's confirmation before updating the logical slot. + */ + if (remote_slot->confirmed_lsn > WalRcv->latestWalEnd) + { + elog(ERROR, "skipping sync of slot \"%s\" as the received slot sync " + "LSN %X/%X is ahead of the standby position %X/%X", + remote_slot->name, + LSN_FORMAT_ARGS(remote_slot->confirmed_lsn), + LSN_FORMAT_ARGS(WalRcv->latestWalEnd)); + + return; + } + + /* Search for the named slot */ + if ((s = SearchNamedReplicationSlot(remote_slot->name, true))) + { + SpinLockAcquire(&s->mutex); + sync_state = s->data.sync_state; + SpinLockRelease(&s->mutex); + } + + StartTransactionCommand(); + + /* + * Already existing slot (created by slot sync worker) and ready for sync, + * acquire and sync it. + */ + if (sync_state == SYNCSLOT_STATE_READY) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* + * Copy the invalidation cause from remote only if local slot is not + * invalidated locally, we don't want to overwrite existing one. + */ + if (MyReplicationSlot->data.invalidated == RS_INVAL_NONE) + { + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.invalidated = remote_slot->invalidated; + SpinLockRelease(&MyReplicationSlot->mutex); + } + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + /* + * With hot_standby_feedback enabled and invalidations handled + * apropriately as above, this should never happen. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn) + { + ereport(ERROR, + errmsg("not synchronizing local slot \"%s\" LSN(%X/%X)" + " to remote slot's LSN(%X/%X) as synchronization " + " would move it backwards", remote_slot->name, + LSN_FORMAT_ARGS(MyReplicationSlot->data.restart_lsn), + LSN_FORMAT_ARGS(remote_slot->restart_lsn))); + + goto cleanup; + } + + if (remote_slot->confirmed_lsn != MyReplicationSlot->data.confirmed_flush || + remote_slot->restart_lsn != MyReplicationSlot->data.restart_lsn || + remote_slot->catalog_xmin != MyReplicationSlot->data.catalog_xmin) + { + /* Update LSN of slot to remote slot's current position */ + local_slot_update(remote_slot); + ReplicationSlotSave(); + *slot_updated = true; + } + } + + /* + * Already existing slot but not ready (i.e. waiting for the primary + * server to catch-up), lets attempt to make it sync-ready now. + */ + else if (sync_state == SYNCSLOT_STATE_INITIATED) + { + ReplicationSlotAcquire(remote_slot->name, true); + + /* Skip the sync if slot has been invalidated locally. */ + if (MyReplicationSlot->data.invalidated != RS_INVAL_NONE) + goto cleanup; + + /* + * Refer the slot creation part (last 'else' block) for more details + * on this wait. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + if (!wait_for_primary_slot_catchup(wrconn, remote_slot, NULL)) + { + goto cleanup; + } + } + + /* + * Wait for primary is over, update the lsns and mark the slot as + * READY for further syncs. + */ + local_slot_update(remote_slot); + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + /* Save the changes */ + ReplicationSlotMarkDirty(); + ReplicationSlotSave(); + + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" is sync-ready now", + remote_slot->name)); + } + /* User created slot with the same name exists, raise ERROR. */ + else if (sync_state == SYNCSLOT_STATE_NONE) + { + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("skipping sync of slot \"%s\" as it is a user created" + " slot", remote_slot->name), + errdetail("This slot has failover enabled on the primary and" + " thus is sync candidate but user created slot with" + " the same name already exists on the standby"))); + } + /* Otherwise create the slot first. */ + else + { + TransactionId xmin_horizon = InvalidTransactionId; + ReplicationSlot *slot; + + ReplicationSlotCreate(remote_slot->name, true, RS_EPHEMERAL, + remote_slot->two_phase, + remote_slot->failover); + slot = MyReplicationSlot; + + SpinLockAcquire(&slot->mutex); + slot->data.database = get_database_oid(remote_slot->database, false); + + namestrcpy(&slot->data.plugin, remote_slot->plugin); + SpinLockRelease(&slot->mutex); + + ReplicationSlotReserveWal(); + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + xmin_horizon = GetOldestSafeDecodingTransactionId(true); + SpinLockAcquire(&slot->mutex); + slot->effective_catalog_xmin = xmin_horizon; + slot->data.catalog_xmin = xmin_horizon; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(true); + LWLockRelease(ProcArrayLock); + + /* + * If the local restart_lsn and/or local catalog_xmin is ahead of + * those on the remote then we cannot create the local slot in sync + * with the primary server because that would mean moving the local + * slot backwards and we might not have WALs retained for old LSN. In + * this case we will wait for the primary server's restart_lsn and + * catalog_xmin to catch up with the local one before attempting the + * sync. + */ + if (remote_slot->restart_lsn < MyReplicationSlot->data.restart_lsn || + TransactionIdPrecedes(remote_slot->catalog_xmin, + MyReplicationSlot->data.catalog_xmin)) + { + bool persist; + + if (!wait_for_primary_slot_catchup(wrconn, remote_slot, &persist)) + { + /* + * The remote slot didn't catch up to locally reserved + * position. + * + * We do not drop the slot because the restart_lsn can be + * ahead of the current location when recreating the slot in + * the next cycle. It may take more time to create such a + * slot. Therefore, we persist it (provided remote-slot is + * still valid) and attempt the wait and synchronization in + * the next cycle. + */ + if (persist) + { + /* Mark it as sync initiated */ + SpinLockAcquire(&MyReplicationSlot->mutex); + slot->data.sync_state = SYNCSLOT_STATE_INITIATED; + SpinLockRelease(&MyReplicationSlot->mutex); + + ReplicationSlotPersist(); + *slot_updated = true; + } + + goto cleanup; + } + } + + + /* + * Wait for primary is either not needed or is over. Update the lsns + * and mark the slot as READY for further syncs. + */ + local_slot_update(remote_slot); + SpinLockAcquire(&MyReplicationSlot->mutex); + MyReplicationSlot->data.sync_state = SYNCSLOT_STATE_READY; + SpinLockRelease(&MyReplicationSlot->mutex); + + /* Mark the slot as PERSISTENT and save the changes to disk */ + ReplicationSlotPersist(); + *slot_updated = true; + + ereport(LOG, errmsg("newly locally created slot \"%s\" is sync-ready now", + remote_slot->name)); + } + +cleanup: + + ReplicationSlotRelease(); + CommitTransactionCommand(); + + return; +} + +/* + * Synchronize slots. + * + * Gets the failover logical slots info from the primary server and update + * the slots locally. Creates the slots if not present on the standby. + * + * Returns nap time for the next sync-cycle. + */ +static long +synchronize_slots(WalReceiverConn *wrconn) +{ +#define SLOTSYNC_COLUMN_COUNT 9 + Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID, + LSNOID, XIDOID, BOOLOID, BOOLOID, TEXTOID, INT2OID}; + + WalRcvExecResult *res; + TupleTableSlot *slot; + StringInfoData s; + List *remote_slot_list = NIL; + MemoryContext oldctx = CurrentMemoryContext; + long naptime = WORKER_DEFAULT_NAPTIME_MS; + ListCell *cell; + bool slot_updated = false; + TimestampTz now; + + /* The primary_slot_name is not set yet or WALs not received yet */ + if (!WalRcv || + (WalRcv->slotname[0] == '\0') || + XLogRecPtrIsInvalid(WalRcv->latestWalEnd)) + return naptime; + + /* The syscache access needs a transaction env. */ + StartTransactionCommand(); + + /* + * Make result tuples live outside TopTransactionContext to make them + * accessible even after transaction is committed. + */ + MemoryContextSwitchTo(oldctx); + + /* Construct query to get slots info from the primary server */ + initStringInfo(&s); + construct_slot_query(&s); + + elog(DEBUG2, "slot sync worker's query:%s \n", s.data); + + /* Execute the query */ + res = walrcv_exec(wrconn, s.data, SLOTSYNC_COLUMN_COUNT, slotRow); + pfree(s.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch failover logical slots info " + "from the primary server: %s", res->err))); + + CommitTransactionCommand(); + + /* Switch to oldctx we saved */ + MemoryContextSwitchTo(oldctx); + + /* Construct the remote_slot tuple and synchronize each slot locally */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + bool isnull; + RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot)); + + remote_slot->name = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + remote_slot->plugin = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + /* + * It is possible to get null values for LSN and Xmin if slot is + * invalidated on the primary server, so handle accordingly. + */ + remote_slot->confirmed_lsn = DatumGetLSN(slot_getattr(slot, 3, &isnull)); + if (isnull) + remote_slot->confirmed_lsn = InvalidXLogRecPtr; + + remote_slot->restart_lsn = DatumGetLSN(slot_getattr(slot, 4, &isnull)); + if (isnull) + remote_slot->restart_lsn = InvalidXLogRecPtr; + + remote_slot->catalog_xmin = DatumGetTransactionId(slot_getattr(slot, + 5, &isnull)); + if (isnull) + remote_slot->catalog_xmin = InvalidTransactionId; + + remote_slot->two_phase = DatumGetBool(slot_getattr(slot, 6, &isnull)); + Assert(!isnull); + + remote_slot->failover = DatumGetBool(slot_getattr(slot, 7, &isnull)); + Assert(!isnull); + + remote_slot->database = TextDatumGetCString(slot_getattr(slot, + 8, &isnull)); + Assert(!isnull); + + remote_slot->invalidated = DatumGetInt16(slot_getattr(slot, 9, &isnull)); + Assert(!isnull); + + /* Create list of remote slots */ + remote_slot_list = lappend(remote_slot_list, remote_slot); + + ExecClearTuple(slot); + } + + /* + * Drop local slots that no longer need to be synced. Do it before + * synchronize_one_slot to allow dropping of slots before actual sync + * which are invalidated locally while still valid on the primary server. + */ + drop_obsolete_slots(remote_slot_list); + + /* Now sync the slots locally */ + foreach(cell, remote_slot_list) + { + RemoteSlot *remote_slot = (RemoteSlot *) lfirst(cell); + + synchronize_one_slot(wrconn, remote_slot, &slot_updated); + } + + now = GetCurrentTimestamp(); + + /* + * If any of the slots get updated in this sync-cycle, retain default + * naptime and update 'last_update_time' in slot sync worker. But if no + * activity is observed in this sync-cycle, then increase naptime provided + * inactivity time reaches threshold. + */ + if (slot_updated) + SlotSyncWorker->last_update_time = now; + + else if (TimestampDifferenceExceeds(SlotSyncWorker->last_update_time, + now, WORKER_INACTIVITY_THRESHOLD_MS)) + naptime = WORKER_INACTIVITY_NAPTIME_MS; + + /* We are done, free remote_slot_list elements */ + list_free_deep(remote_slot_list); + + walrcv_clear_result(res); + + return naptime; +} + +/* + * Connect to the remote (primary) server. + * + * This uses GUC primary_conninfo in order to connect to the primary. + * For slot sync to work, primary_conninfo is required to specify dbname + * as well. + */ +static WalReceiverConn * +remote_connect(void) +{ + WalReceiverConn *wrconn = NULL; + char *err; + + wrconn = walrcv_connect(PrimaryConnInfo, true, false, + cluster_name[0] ? cluster_name : "slotsyncworker", &err); + if (wrconn == NULL) + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not connect to the primary server: %s", err))); + return wrconn; +} + +/* + * Re-read the config file. + * + * If primary_conninfo has changed, reconnect to primary. + */ +static void +slotsync_reread_config(WalReceiverConn **wrconn) +{ + char *conninfo = pstrdup(PrimaryConnInfo); + + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Exit if GUC primary_conninfo got changed, let the launcher relaunch it */ + if (strcmp(conninfo, PrimaryConnInfo) != 0) + { + if (*wrconn) + walrcv_disconnect(*wrconn); + + pfree(conninfo); + + ereport(LOG, + (errmsg("replication slot sync worker will stop because the " + "primary_conninfo has changed"))); + proc_exit(0); + } + + pfree(conninfo); +} + +/* + * Interrupt handler for main loop of slot sync worker. + */ +static void +ProcessSlotSyncInterrupts(WalReceiverConn **wrconn) +{ + CHECK_FOR_INTERRUPTS(); + + if (ShutdownRequestPending) + { + ereport(LOG, + errmsg("replication slot sync worker is shutting" + " down on receiving SIGINT")); + + walrcv_disconnect(*wrconn); + proc_exit(0); + } + + + if (ConfigReloadPending) + slotsync_reread_config(wrconn); +} + +/* + * The main loop of our worker process. + */ +void +ReplSlotSyncWorkerMain(Datum main_arg) +{ + WalReceiverConn *wrconn = NULL; + char *dbname; + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + slotsync_worker_attach(); + + /* + * If the standby has been promoted, skip the slot synchronization + * process. + * + * Although the startup process stops all the slot sync workers on + * promotion, the launcher may not have realized the promotion and could + * start additional workers after that. Therefore, this check is still + * necessary to prevent these additional workers from running. + */ + if (PromoteIsTriggered()) + exit(0); + + ereport(LOG, errmsg("replication slot sync worker started")); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + /* + * Get the user provided dbname from the connection string, if dbname not + * provided, skip sync. + */ + dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo); + if (dbname == NULL) + proc_exit(0); + + /* + * Connect to the database specified by user in PrimaryConnInfo. We need a + * database connection for walrcv_exec to work. Please see comments atop + * libpqrcv_exec. + */ + BackgroundWorkerInitializeConnection(dbname, NULL, 0); + + /* Connect to the primary server */ + wrconn = remote_connect(); + + /* Main wait loop. */ + for (;;) + { + int rc; + long naptime; + + ProcessSlotSyncInterrupts(&wrconn); + + /* Check if got promoted */ + if (!RecoveryInProgress()) + { + /* + * Drop the slots for which sync is initiated but not yet + * completed i.e. they are still waiting for the primary server to + * catch up. + */ + slotsync_drop_initiated_slots(); + ereport(LOG, + errmsg("exiting slot sync woker on promotion of standby")); + proc_exit(0); + } + + naptime = synchronize_slots(wrconn); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_REPL_SLOTSYNC_MAIN); + + if (rc & WL_LATCH_SET) + ResetLatch(MyLatch); + } + + /* + * The slot sync worker can not get here because it will only stop when it + * receives a SIGINT from the logical replication launcher, or when there + * is an error. + */ + Assert(false); +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f99c12513d..ddd41a4142 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -100,6 +100,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/subscriptioncmds.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -246,7 +247,7 @@ wait_for_worker_state_change(char expected_state) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, false); - if (worker && worker->proc) + if (worker && worker->hdr.proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); if (!worker) @@ -535,7 +536,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) if (rstate->state == SUBREL_STATE_SYNCWAIT) { /* Signal the sync worker, as it may be waiting for us. */ - if (syncworker->proc) + if (syncworker->hdr.proc) logicalrep_worker_wakeup_ptr(syncworker); /* Now safe to release the LWLock */ diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 85dd935e65..b903b90140 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -47,6 +47,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "replication/slot.h" +#include "replication/walsender.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/proc.h" @@ -318,6 +319,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, slot->data.two_phase = two_phase; slot->data.two_phase_at = InvalidXLogRecPtr; slot->data.failover = failover; + slot->data.sync_state = SYNCSLOT_STATE_NONE; /* and then data only present in shared memory */ slot->just_dirtied = false; @@ -677,12 +679,26 @@ restart: * Permanently drop replication slot identified by the passed in name. */ void -ReplicationSlotDrop(const char *name, bool nowait) +ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd) { Assert(MyReplicationSlot == NULL); ReplicationSlotAcquire(name, nowait); + /* + * Do not allow users to drop the slots which are currently being synced + * from the primary to the standby. + */ + if (user_cmd && RecoveryInProgress() && + MyReplicationSlot->data.sync_state != SYNCSLOT_STATE_NONE) + { + ReplicationSlotRelease(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop replication slot \"%s\"", name), + errdetail("This slot is being synced from the primary."))); + } + ReplicationSlotDropAcquired(); } diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index fb6e37d2c3..7b1e0c1552 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -226,11 +226,38 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) CheckSlotRequirements(); - ReplicationSlotDrop(NameStr(*name), true); + ReplicationSlotDrop(NameStr(*name), true, true); PG_RETURN_VOID(); } +/* + * SQL function for getting invalidation cause of a slot. + * + * Returns ReplicationSlotInvalidationCause enum value for valid slot_name; + * returns NULL if slot with given name is not found. + * + * Returns RS_INVAL_NONE if the given slot is not invalidated. + */ +Datum +pg_get_slot_invalidation_cause(PG_FUNCTION_ARGS) +{ + Name name = PG_GETARG_NAME(0); + ReplicationSlot *s; + ReplicationSlotInvalidationCause cause; + + s = SearchNamedReplicationSlot(NameStr(*name), true); + + if (s == NULL) + PG_RETURN_NULL(); + + SpinLockAcquire(&s->mutex); + cause = s->data.invalidated; + SpinLockRelease(&s->mutex); + + PG_RETURN_INT16(cause); +} + /* * pg_get_replication_slots - SQL SRF showing all replication slots * that currently exist on the database cluster. @@ -238,7 +265,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 16 +#define PG_GET_REPLICATION_SLOTS_COLS 17 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; XLogRecPtr currlsn; int slotno; @@ -420,6 +447,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) values[i++] = BoolGetDatum(slot_contents.data.failover); + values[i++] = CharGetDatum(slot_contents.data.sync_state); + Assert(i == PG_GET_REPLICATION_SLOTS_COLS); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 6c062eef16..8e4b0c66be 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1255,7 +1255,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - ReplicationSlotDrop(cmd->slotname, !cmd->wait); + ReplicationSlotDrop(cmd->slotname, !cmd->wait, false); } /* diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e62a3f1bc0 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +SlotSyncWorkerLock 49 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index ede94a1ede..7eb735824c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -53,6 +53,8 @@ LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPL_SLOTSYNC_MAIN "Waiting in main loop of slot sync worker." +REPL_SLOTSYNC_PRIMARY_CATCHUP "Waiting for the primary to catch-up, in slot sync worker." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index be05790ff3..a61d6d84e0 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -65,8 +65,11 @@ #include "postmaster/syslogger.h" #include "postmaster/walwriter.h" #include "replication/logicallauncher.h" +#include "replication/reorderbuffer.h" #include "replication/slot.h" #include "replication/syncrep.h" +#include "replication/walreceiver.h" +#include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" @@ -2021,6 +2024,15 @@ struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_syncslot", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Enables a physical standby to synchronize logical failover slots from the primary server."), + }, + &enable_syncslot, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index dd2769cdd3..152e8bff64 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -355,6 +355,7 @@ #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery +#enable_syncslot = on # enables slot synchronization on the physical standby from the primary # - Subscribers - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d906734750..6a8192ad83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11095,14 +11095,18 @@ proname => 'pg_drop_replication_slot', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'name', prosrc => 'pg_drop_replication_slot' }, +{ oid => '8484', descr => 'what caused the replication slot to become invalid', + proname => 'pg_get_slot_invalidation_cause', provolatile => 's', proisstrict => 't', + prorettype => 'int2', proargtypes => 'name', + prosrc => 'pg_get_slot_invalidation_cause' }, { oid => '3781', descr => 'information about replication slots currently in use', proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8,bool,bool,bool,char}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,safe_wal_size,two_phase,conflicting,failover,sync_state}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 214dc6c29e..75b4b2040d 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -17,6 +17,7 @@ #include "catalog/objectaddress.h" #include "parser/parse_node.h" +#include "replication/walreceiver.h" extern ObjectAddress CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bool isTopLevel); @@ -28,4 +29,7 @@ extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); extern char defGetStreamingMode(DefElem *def); +extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, + char *slotname, bool missing_ok); + #endif /* SUBSCRIPTIONCMDS_H */ diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index bef8a7162e..085d729620 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -33,7 +33,6 @@ typedef struct IdentifySystemCmd NodeTag type; } IdentifySystemCmd; - /* ---------------------- * BASE_BACKUP command * ---------------------- diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index a07c9cb311..02499a7e66 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -15,9 +15,11 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; +extern PGDLLIMPORT bool enable_syncslot; + extern void ApplyLauncherRegister(void); -extern void ApplyLauncherMain(Datum main_arg); +extern void LauncherMain(Datum main_arg); extern Size ApplyLauncherShmemSize(void); extern void ApplyLauncherShmemInit(void); @@ -31,4 +33,9 @@ extern bool IsLogicalLauncher(void); extern pid_t GetLeaderApplyWorkerPid(pid_t pid); +extern void ShutDownSlotSync(void); + +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; + #endif /* LOGICALLAUNCHER_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index bbd71d0b42..baad5a8f3a 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -19,6 +19,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); extern void TablesyncWorkerMain(Datum main_arg); +extern void ReplSlotSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index a92fb38ec0..9e7a4f695a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -15,7 +15,6 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "replication/walreceiver.h" /* * Behaviour of replication slots, upon release or crash. @@ -52,6 +51,14 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_WAL_LEVEL, } ReplicationSlotInvalidationCause; +/* The possible values for 'sync_state' in ReplicationSlotPersistentData */ +#define SYNCSLOT_STATE_NONE 'n' /* None for user created slots */ +#define SYNCSLOT_STATE_INITIATED 'i' /* Sync initiated for the slot but + * not completed yet, waiting for + * the primary server to catch-up */ +#define SYNCSLOT_STATE_READY 'r' /* Initialization complete, ready + * to be synced further */ + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -112,6 +119,13 @@ typedef struct ReplicationSlotPersistentData /* plugin name */ NameData plugin; + /* + * Is this a slot created by a sync-slot worker? + * + * Relevant for logical slots on the physical standby. + */ + char sync_state; + /* * Is this a failover slot (sync candidate for physical standbys)? * Relevant for logical slots on the primary server. @@ -227,7 +241,7 @@ extern void ReplicationSlotCreate(const char *name, bool db_specific, ReplicationSlotPersistency persistency, bool two_phase, bool failover); extern void ReplicationSlotPersist(void); -extern void ReplicationSlotDrop(const char *name, bool nowait); +extern void ReplicationSlotDrop(const char *name, bool nowait, bool user_cmd); extern void ReplicationSlotAlter(const char *name, bool failover); extern void ReplicationSlotAcquire(const char *name, bool nowait); @@ -253,7 +267,6 @@ extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_l extern int ReplicationSlotIndex(ReplicationSlot *slot); extern bool ReplicationSlotName(int index, Name name); extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot); -extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(bool is_shutdown); @@ -261,7 +274,6 @@ extern void CheckPointReplicationSlots(bool is_shutdown); extern void CheckSlotRequirements(void); extern void CheckSlotPermissions(void); -extern void WaitForStandbyLSN(XLogRecPtr wait_for_lsn); extern List *GetStandbySlotList(bool copy); #endif /* SLOT_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 115344f1c4..9825c7ce16 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -20,6 +20,7 @@ #include "pgtime.h" #include "port/atomics.h" #include "replication/logicalproto.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/latch.h" @@ -280,6 +281,21 @@ typedef void (*walrcv_get_senderinfo_fn) (WalReceiverConn *conn, typedef char *(*walrcv_identify_system_fn) (WalReceiverConn *conn, TimeLineID *primary_tli); +/* + * walrcv_get_dbinfo_for_failover_slots_fn + * + * Run LIST_DBID_FOR_FAILOVER_SLOTS on primary server to get the + * list of unique DBIDs for failover logical slots + */ +typedef List *(*walrcv_get_dbinfo_for_failover_slots_fn) (WalReceiverConn *conn); + +/* + * walrcv_get_dbname_from_conninfo_fn + * + * Returns the dbid from the primary_conninfo + */ +typedef char *(*walrcv_get_dbname_from_conninfo_fn) (const char *conninfo); + /* * walrcv_server_version_fn * @@ -404,6 +420,7 @@ typedef struct WalReceiverFunctionsType walrcv_get_conninfo_fn walrcv_get_conninfo; walrcv_get_senderinfo_fn walrcv_get_senderinfo; walrcv_identify_system_fn walrcv_identify_system; + walrcv_get_dbname_from_conninfo_fn walrcv_get_dbname_from_conninfo; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; walrcv_startstreaming_fn walrcv_startstreaming; @@ -429,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_get_senderinfo(conn, sender_host, sender_port) #define walrcv_identify_system(conn, primary_tli) \ WalReceiverFunctions->walrcv_identify_system(conn, primary_tli) +#define walrcv_get_dbname_from_conninfo(conninfo) \ + WalReceiverFunctions->walrcv_get_dbname_from_conninfo(conninfo) #define walrcv_server_version(conn) \ WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index a9bba11187..0a0d7a2a97 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -1,7 +1,8 @@ /*------------------------------------------------------------------------- * * worker_internal.h - * Internal headers shared by logical replication workers. + * Internal headers shared by logical replication workers + * and slotsync workers. * * Portions Copyright (c) 2016-2023, PostgreSQL Global Development Group * @@ -36,14 +37,9 @@ typedef enum LogicalRepWorkerType WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; -typedef struct LogicalRepWorker +/* Common data for Slotsync and LogicalRep workers */ +typedef struct LogicalWorkerHeader { - /* What type of worker is this? */ - LogicalRepWorkerType type; - - /* Time at which this worker was launched. */ - TimestampTz launch_time; - /* Indicates if this slot is used or free. */ bool in_use; @@ -53,6 +49,19 @@ typedef struct LogicalRepWorker /* Pointer to proc array. NULL if not running. */ PGPROC *proc; +} LogicalWorkerHeader; + +/* Shared memory structure for logical replication workers. */ +typedef struct LogicalRepWorker +{ + LogicalWorkerHeader hdr; + + /* Time at which this worker was launched. */ + TimestampTz launch_time; + + /* What type of worker is this? */ + LogicalRepWorkerType type; + /* Database id to connect to. */ Oid dbid; @@ -96,6 +105,32 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* + * Shared memory structure for Slot-Sync worker. It is allocated by logical + * replication launcher and then read by each slot sync worker. + * + * It is protected by LWLock (SlotSyncWorkerLock). Each slot sync worker + * reading the structure needs to hold the lock in shared mode, whereas + * the logical replication launcher which updates it needs to hold the lock + * in exclusive mode. + */ +typedef struct SlotSyncWorkerInfo +{ + /* + * The header used by slot-sync worker and logical rep worker. + * + * The header has 'generation'. For slot sync worker, it is not needed + * but since slot sync worker uses some of the code infrastructure + * provided by logical rep worker which needs 'generation', it is + * thus retained for slot sync worker but is always kept as 0. + */ + LogicalWorkerHeader hdr; + + /* The last sync-cycle time when the worker updated any of the slots. */ + TimestampTz last_update_time; + +} SlotSyncWorkerInfo; + /* * State of the transaction in parallel apply worker. * @@ -234,12 +269,16 @@ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; /* Worker and subscription objects. */ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; +extern PGDLLIMPORT SlotSyncWorkerInfo *SlotSyncWorker; extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern void logicalrep_worker_attach(int slot); +extern void slotsync_worker_attach(void); +extern void slotsync_worker_detach(int code, Datum arg); +extern void slotsync_drop_initiated_slots(void); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); @@ -329,9 +368,9 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); -#define isParallelApplyWorker(worker) ((worker)->in_use && \ +#define isParallelApplyWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTablesyncWorker(worker) ((worker)->hdr.in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) static inline bool @@ -343,14 +382,14 @@ am_tablesync_worker(void) static inline bool am_leader_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return (MyLogicalRepWorker->type == WORKERTYPE_APPLY); } static inline bool am_parallel_apply_worker(void) { - Assert(MyLogicalRepWorker->in_use); + Assert(MyLogicalRepWorker->hdr.in_use); return isParallelApplyWorker(MyLogicalRepWorker); } diff --git a/src/test/recovery/t/050_verify_slot_order.pl b/src/test/recovery/t/050_verify_slot_order.pl index 42e51634c5..d26ba41c4d 100644 --- a/src/test/recovery/t/050_verify_slot_order.pl +++ b/src/test/recovery/t/050_verify_slot_order.pl @@ -142,4 +142,131 @@ $result = $subscriber1->safe_psql('postgres', "SELECT count(*) = $primary_row_count FROM tab_int;"); is($result, 't', "subscriber1 gets data from primary after standby1 acknowledges changes"); +# Test logical failover slots on the standby +# Configure standby3 to replicate and synchronize logical slots configured +# for failover on the primary +# +# failover slot lsub1_slot->| ----> subscriber1 (connected via logical replication) +# primary ---> | +# physical slot sb3_slot--->| ----> standby3 (connected via streaming replication) +# | lsub1_slot(synced_slot) + +# Cleanup old standby_slot_names +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = '' +)); +$primary->start; + +$primary->psql('postgres', + q{SELECT pg_create_physical_replication_slot('sb3_slot');}); + +$backup_name = 'backup2'; +$primary->backup($backup_name); + +# Create standby3 +my $standby3 = PostgreSQL::Test::Cluster->new('standby3'); +$standby3->init_from_backup( + $primary, $backup_name, + has_streaming => 1, + has_restoring => 1); + +my $connstr_1 = $primary->connstr; +$standby3->stop; +$standby3->append_conf( + 'postgresql.conf', q{ +enable_syncslot = true +hot_standby_feedback = on +primary_slot_name = 'sb3_slot' +}); +$standby3->append_conf( + 'postgresql.conf', qq( +primary_conninfo = '$connstr_1 dbname=postgres' +)); +$standby3->start; + +# Add this standby into the primary's configuration +$primary->stop; +$primary->append_conf( + 'postgresql.conf', qq( +standby_slot_names = 'sb3_slot' +)); +$primary->start; + +# Restart the standby +$standby3->restart; + +# Wait for the standby to start sync +my $offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? waiting for remote slot \"lsub1_slot\"/, + $offset); + +# Advance lsn on the primary +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); +$primary->safe_psql('postgres', + "SELECT pg_log_standby_snapshot();"); + +# Wait for the standby to finish sync +$offset = -s $standby3->logfile; +$standby3->wait_for_log( + qr/LOG: ( [A-Z0-9]+:)? wait over for remote slot \"lsub1_slot\"/, + $offset); + +# Confirm that logical failover slot is created on the standby +is( $standby3->safe_psql('postgres', + q{SELECT slot_name FROM pg_replication_slots;} + ), + 'lsub1_slot', + 'failover slot was created'); + +# Verify slot properties on the standby +is( $standby3->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|r", + 'logical slot has sync_state as ready and failover as true on standby'); + +# Verify slot properties on the primary +is( $primary->safe_psql('postgres', + q{SELECT failover, sync_state FROM pg_replication_slots WHERE slot_name = 'lsub1_slot';} + ), + "t|n", + 'logical slot has sync_state as none and failover as true on primary'); + +# Test to confirm that restart_lsn of the logical slot on the primary is synced to the standby + +# Truncate table on primary +$primary->safe_psql('postgres', + "TRUNCATE TABLE tab_int;"); + +# Insert data on the primary +$primary->safe_psql('postgres', + "INSERT INTO tab_int SELECT generate_series(1, $primary_row_count);"); + +# let the slots get synced on the standby +sleep 2; + +# Get the restart_lsn for the logical slot lsub1_slot on the primary +my $primary_lsn = $primary->safe_psql('postgres', + "SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that restart_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= restart_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'restart_lsn of slot lsub1_slot synced to standby'); + +# Get the confirmed_flush_lsn for the logical slot lsub1_slot on the primary +$primary_lsn = $primary->safe_psql('postgres', + "SELECT confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';"); + +# Confirm that confirmed_flush_lsn of lsub1_slot slot is synced to the standby +$result = $standby3->safe_psql('postgres', + qq[SELECT '$primary_lsn' <= confirmed_flush_lsn from pg_replication_slots WHERE slot_name = 'lsub1_slot';]); +is($result, 't', 'confirmed_flush_lsn of slot lsub1_slot synced to the standby'); + done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c9647e86b2..dab989b520 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1474,8 +1474,9 @@ pg_replication_slots| SELECT l.slot_name, l.safe_wal_size, l.two_phase, l.conflicting, - l.failover - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover) + l.failover, + l.sync_state + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, safe_wal_size, two_phase, conflicting, failover, sync_state) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 271313ebf8..aac83755de 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -132,8 +132,9 @@ select name, setting from pg_settings where name like 'enable%'; enable_self_join_removal | on enable_seqscan | on enable_sort | on + enable_syncslot | off enable_tidscan | on -(22 rows) +(23 rows) -- There are always wait event descriptions for various types. select type, count(*) > 0 as ok FROM pg_wait_events diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bc057c74d8..be5061fd13 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1431,6 +1431,7 @@ LimitState LimitStateCond List ListCell +ListDBForLogicalSlotsCmd ListDictionary ListParsedLex ListenAction @@ -1510,6 +1511,7 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet +LogicalWorkerHeader LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus @@ -2313,6 +2315,7 @@ RelocationBufferInfo RelptrFreePageBtree RelptrFreePageManager RelptrFreePageSpanLeader +RemoteSlot RenameStmt ReopenPtrType ReorderBuffer @@ -2571,6 +2574,8 @@ SlabBlock SlabContext SlabSlot SlotNumber +SlotSyncWorker +SlotSyncWorkerInfo SlruCtl SlruCtlData SlruErrorCause -- 2.34.1 ^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Synchronizing slots from primary to standby @ 2023-11-28 10:02 shveta malik <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 55+ messages in thread From: shveta malik @ 2023-11-28 10:02 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; Bharath Rupireddy <[email protected]>; Peter Eisentraut <[email protected]>; Bruce Momjian <[email protected]>; Ashutosh Sharma <[email protected]>; Andres Freund <[email protected]>; pgsql-hackers; Ajin Cherian <[email protected]>; Alvaro Herrera <[email protected]>; shveta malik <[email protected]> On Fri, Nov 17, 2023 at 5:08 PM Amit Kapila <[email protected]> wrote: > > On Thu, Nov 16, 2023 at 5:34 PM shveta malik <[email protected]> wrote: > > > > PFA v35. > > > > Review v35-0002* > ============== > 1. > As quoted in the commit message, > > > If a logical slot is invalidated on the primary, slot on the standby is also > invalidated. If a logical slot on the primary is valid but is invalidated > on the standby due to conflict (say required rows removed on the primary), > then that slot is dropped and recreated on the standby in next sync-cycle. > It is okay to recreate such slots as long as these are not consumable on the > standby (which is the case currently). > > > > I think this won't happen normally because of the physical slot and > hot_standby_feedback but probably can occur in cases like if the user > temporarily switches hot_standby_feedback from on to off. Are there > any other reasons? I think we can mention the cases along with it as > well at least for now. Additionally, I think this should be covered in > code comments as well. > > 2. > #include "postgres.h" > - > +#include "access/genam.h" > > Spurious line removal. > > 3. > A password needs to be provided too, if the sender demands password > authentication. It can be provided in the > <varname>primary_conninfo</varname> string, or in a separate > - <filename>~/.pgpass</filename> file on the standby server (use > - <literal>replication</literal> as the database name). > - Do not specify a database name in the > - <varname>primary_conninfo</varname> string. > + <filename>~/.pgpass</filename> file on the standby server. > + </para> > + <para> > + Specify <literal>dbname</literal> in > + <varname>primary_conninfo</varname> string to allow synchronization > + of slots from the primary server to the standby server. > + This will only be used for slot synchronization. It is ignored > + for streaming. > > Is there a reason to remove part of the earlier sentence "use > <literal>replication</literal> as the database name"? > > 4. > + <primary><varname>enable_syncslot</varname> configuration > parameter</primary> > + </indexterm> > + </term> > + <listitem> > + <para> > + It enables a physical standby to synchronize logical failover slots > + from the primary server so that logical subscribers are not blocked > + after failover. > + </para> > + <para> > + It is enabled by default. This parameter can only be set in the > + <filename>postgresql.conf</filename> file or on the server > command line. > + </para> > > I think you forgot to update the documentation for the default value > of this variable. > > 5. > + * a) start the logical replication workers for every enabled subscription > + * when not in standby_mode > + * b) start the slot-sync worker for logical failover slots synchronization > + * from the primary server when in standby_mode. > > Either use a full stop after both lines or none of these. > > 6. > +static void slotsync_worker_cleanup(SlotSyncWorkerInfo * worker); > > There shouldn't be space between * and the worker. > > 7. > + if (!SlotSyncWorker->hdr.in_use) > + { > + LWLockRelease(SlotSyncWorkerLock); > + ereport(ERROR, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg("replication slot-sync worker not initialized, " > + "cannot attach"))); > + } > + > + if (SlotSyncWorker->hdr.proc) > + { > + LWLockRelease(SlotSyncWorkerLock); > + ereport(ERROR, > + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), > + errmsg("replication slot-sync worker is " > + "already running, cannot attach"))); > + } > > Using slot-sync in the error messages looks a bit odd to me. Can we > use "replication slot sync worker ..." in both these and other > similar messages? I think it would be better if we don't split the > messages into multiple lines in these cases as messages don't appear > too long to me. > > 8. > +/* > + * Detach the worker from DSM and update 'proc' and 'in_use'. > + * Logical replication launcher will come to know using these > + * that the worker has shutdown. > + */ > +void > +slotsync_worker_detach(int code, Datum arg) > +{ > > I think the reference to DSM is leftover from the previous version of > the patch. Can we change the above comments as per the new code? > > 9. > +static bool > +slotsync_worker_launch() > { > ... > + /* TODO: do we really need 'generation', analyse more here */ > + worker->hdr.generation++; > > We should do something about this TODO. As per my understanding, we > don't need a generation number for the slot sync worker as we have one > such worker but I guess the patch requires it because we are using > existing logical replication worker infrastructure. This brings the > question of whether we really need a separate SlotSyncWorkerInfo or if > we can use existing LogicalRepWorker and distinguish it with > LogicalRepWorkerType? I guess you didn't use it because most of the > fields in LogicalRepWorker will be unused for slot sync worker. > > 10. > + * Can't use existing functions like 'get_database_oid' from dbcommands.c for > + * validity purpose as they need db connection. > + */ > +static bool > +validate_dbname(const char *dbname) > > I don't know how important it is to validate the dbname before > launching the sync slot worker because anyway after launching, it will > give an error while initializing the connection if the dbname is > invalid. But, if we think it is really required, did you consider > using GetDatabaseTuple()? I have removed 'validate_dbname' in v40. We let dbname go through BackgroundWorkerInitializeConnection() which internally does dbname validation. Later if 'primary_conninfo' is changed and the db name specified in it is different, we exit the worker and let it get restarted which will do the validation again when it does BackgroundWorkerInitializeConnection(). > > -- > With Regards, > Amit Kapila. ^ permalink raw reply [nested|flat] 55+ messages in thread
end of thread, other threads:[~2023-11-28 10:02 UTC | newest] Thread overview: 55+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-07-17 14:31 [PATCH 3/3] Aggregate push-down - basic functionality. Antonin Houska <[email protected]> 2023-07-25 23:09 [PATCH v8 1/1] add to_binary() and to_oct() Nathan Bossart <[email protected]> 2023-10-27 15:13 Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-10-31 09:37 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-07 10:21 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-07 10:55 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-07 14:28 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-08 03:50 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-08 07:02 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-08 08:57 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-08 09:49 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-08 11:50 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-08 14:39 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-09 02:41 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-09 03:25 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-09 10:54 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-09 15:45 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-10 03:31 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-10 11:00 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-10 07:59 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-13 00:49 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-13 04:00 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-13 05:32 ` Re: Synchronizing slots from primary to standby Ajin Cherian <[email protected]> 2023-11-13 06:38 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-13 09:12 ` Re: Synchronizing slots from primary to standby Ajin Cherian <[email protected]> 2023-11-13 04:24 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-13 09:53 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-09 13:59 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-10 05:41 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-10 07:20 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-10 07:55 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-10 08:15 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-13 13:57 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-14 14:26 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-15 11:51 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-16 10:13 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-16 12:03 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-17 11:38 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-17 11:38 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-21 04:31 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-21 06:01 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-21 08:26 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-28 10:02 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-18 07:21 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-16 05:13 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-17 01:46 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-17 11:48 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-18 10:45 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-20 09:47 ` Re: Synchronizing slots from primary to standby Drouvot, Bertrand <[email protected]> 2023-11-20 10:58 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-21 04:28 ` Re: Synchronizing slots from primary to standby Amit Kapila <[email protected]> 2023-11-23 08:59 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-21 04:31 ` RE: Synchronizing slots from primary to standby Zhijie Hou (Fujitsu) <[email protected]> 2023-11-21 05:45 ` Re: Synchronizing slots from primary to standby shveta malik <[email protected]> 2023-11-17 04:20 ` Re: Synchronizing slots from primary to standby Ajin Cherian <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox