public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 3/4] f!convert the other verbose to int, too
99+ messages / 13 participants
[nested] [flat]
* [PATCH 3/4] f!convert the other verbose to int, too
@ 2021-07-15 08:19 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: Justin Pryzby @ 2021-07-15 08:19 UTC (permalink / raw)
---
src/bin/psql/command.c | 82 ++++++++++----------
src/bin/psql/describe.c | 163 ++++++++++++++++++++--------------------
src/bin/psql/describe.h | 58 +++++++-------
3 files changed, 151 insertions(+), 152 deletions(-)
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index b695e09017d..5f4cde03250 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -76,7 +76,7 @@ static backslashResult exec_command_d(PsqlScanState scan_state, bool active_bran
const char *cmd);
static bool exec_command_dfo(PsqlScanState scan_state, const char *cmd,
const char *pattern,
- bool show_verbose, bool show_system);
+ int verbose, bool show_system);
static backslashResult exec_command_edit(PsqlScanState scan_state, bool active_branch,
PQExpBuffer query_buf, PQExpBuffer previous_buf);
static backslashResult exec_command_ef_ev(PsqlScanState scan_state, bool active_branch,
@@ -756,8 +756,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
{
char *pattern;
int verbose = 0;
- bool show_verbose,
- show_system;
+ bool show_system;
/* We don't do SQLID reduction on the pattern yet */
pattern = psql_scan_slash_option(scan_state,
@@ -766,7 +765,6 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
for (const char *t = cmd; *t != '\0'; ++t)
verbose += *t == '+' ? 1 : 0;
- show_verbose = (bool) (verbose != 0);
show_system = strchr(cmd, 'S') ? true : false;
switch (cmd[1])
@@ -775,10 +773,10 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
case '+':
case 'S':
if (pattern)
- success = describeTableDetails(pattern, show_verbose, show_system);
+ success = describeTableDetails(pattern, verbose, show_system);
else
/* standard listing of interesting things */
- success = listTables("tvmsE", NULL, show_verbose, show_system);
+ success = listTables("tvmsE", NULL, verbose, show_system);
break;
case 'A':
{
@@ -794,16 +792,16 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
success = describeAccessMethods(pattern, verbose);
break;
case 'c':
- success = listOperatorClasses(pattern, pattern2, show_verbose);
+ success = listOperatorClasses(pattern, pattern2, verbose);
break;
case 'f':
- success = listOperatorFamilies(pattern, pattern2, show_verbose);
+ success = listOperatorFamilies(pattern, pattern2, verbose);
break;
case 'o':
- success = listOpFamilyOperators(pattern, pattern2, show_verbose);
+ success = listOpFamilyOperators(pattern, pattern2, verbose);
break;
case 'p':
- success = listOpFamilyFunctions(pattern, pattern2, show_verbose);
+ success = listOpFamilyFunctions(pattern, pattern2, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -814,7 +812,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
}
break;
case 'a':
- success = describeAggregates(pattern, show_verbose, show_system);
+ success = describeAggregates(pattern, verbose, show_system);
break;
case 'b':
success = describeTablespaces(pattern, verbose);
@@ -822,15 +820,15 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
case 'c':
if (strncmp(cmd, "dconfig", 7) == 0)
success = describeConfigurationParameters(pattern,
- show_verbose,
+ verbose,
show_system);
else
success = listConversions(pattern,
- show_verbose,
+ verbose,
show_system);
break;
case 'C':
- success = listCasts(pattern, show_verbose);
+ success = listCasts(pattern, verbose);
break;
case 'd':
if (strncmp(cmd, "ddp", 3) == 0)
@@ -839,7 +837,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
success = objectDescription(pattern, show_system);
break;
case 'D':
- success = listDomains(pattern, show_verbose, show_system);
+ success = listDomains(pattern, verbose, show_system);
break;
case 'f': /* function subsystem */
switch (cmd[2])
@@ -853,7 +851,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
case 't':
case 'w':
success = exec_command_dfo(scan_state, cmd, pattern,
- show_verbose, show_system);
+ verbose, show_system);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -862,23 +860,23 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
break;
case 'g':
/* no longer distinct from \du */
- success = describeRoles(pattern, show_verbose, show_system);
+ success = describeRoles(pattern, verbose, show_system);
break;
case 'l':
- success = listLargeObjects(show_verbose);
+ success = listLargeObjects(verbose);
break;
case 'L':
- success = listLanguages(pattern, show_verbose, show_system);
+ success = listLanguages(pattern, verbose, show_system);
break;
case 'n':
success = listSchemas(pattern, verbose, show_system);
break;
case 'o':
success = exec_command_dfo(scan_state, cmd, pattern,
- show_verbose, show_system);
+ verbose, show_system);
break;
case 'O':
- success = listCollations(pattern, show_verbose, show_system);
+ success = listCollations(pattern, verbose, show_system);
break;
case 'p':
success = permissionsList(pattern, show_system);
@@ -892,7 +890,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
case 't':
case 'i':
case 'n':
- success = listPartitionedTables(&cmd[2], pattern, show_verbose);
+ success = listPartitionedTables(&cmd[2], pattern, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -901,7 +899,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
}
break;
case 'T':
- success = describeTypes(pattern, show_verbose, show_system);
+ success = describeTypes(pattern, verbose, show_system);
break;
case 't':
case 'v':
@@ -909,7 +907,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
case 'i':
case 's':
case 'E':
- success = listTables(&cmd[1], pattern, show_verbose, show_system);
+ success = listTables(&cmd[1], pattern, verbose, show_system);
break;
case 'r':
if (cmd[2] == 'd' && cmd[3] == 's')
@@ -930,36 +928,36 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
switch (cmd[2])
{
case 'p':
- if (show_verbose)
+ if (verbose > 0)
success = describePublications(pattern);
else
success = listPublications(pattern);
break;
case 's':
- success = describeSubscriptions(pattern, show_verbose);
+ success = describeSubscriptions(pattern, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
}
break;
case 'u':
- success = describeRoles(pattern, show_verbose, show_system);
+ success = describeRoles(pattern, verbose, show_system);
break;
case 'F': /* text search subsystem */
switch (cmd[2])
{
case '\0':
case '+':
- success = listTSConfigs(pattern, show_verbose);
+ success = listTSConfigs(pattern, verbose);
break;
case 'p':
- success = listTSParsers(pattern, show_verbose);
+ success = listTSParsers(pattern, verbose);
break;
case 'd':
- success = listTSDictionaries(pattern, show_verbose);
+ success = listTSDictionaries(pattern, verbose);
break;
case 't':
- success = listTSTemplates(pattern, show_verbose);
+ success = listTSTemplates(pattern, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -970,16 +968,16 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
switch (cmd[2])
{
case 's':
- success = listForeignServers(pattern, show_verbose);
+ success = listForeignServers(pattern, verbose);
break;
case 'u':
- success = listUserMappings(pattern, show_verbose);
+ success = listUserMappings(pattern, verbose);
break;
case 'w':
- success = listForeignDataWrappers(pattern, show_verbose);
+ success = listForeignDataWrappers(pattern, verbose);
break;
case 't':
- success = listForeignTables(pattern, show_verbose);
+ success = listForeignTables(pattern, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -987,7 +985,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
}
break;
case 'x': /* Extensions */
- if (show_verbose)
+ if (verbose > 0)
success = listExtensionContents(pattern);
else
success = listExtensions(pattern);
@@ -996,7 +994,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
success = listExtendedStats(pattern);
break;
case 'y': /* Event Triggers */
- success = listEventTriggers(pattern, show_verbose);
+ success = listEventTriggers(pattern, verbose);
break;
default:
status = PSQL_CMD_UNKNOWN;
@@ -1017,7 +1015,7 @@ exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd)
static bool
exec_command_dfo(PsqlScanState scan_state, const char *cmd,
const char *pattern,
- bool show_verbose, bool show_system)
+ int verbose, bool show_system)
{
bool success;
char *arg_patterns[FUNC_MAX_ARGS];
@@ -1040,11 +1038,11 @@ exec_command_dfo(PsqlScanState scan_state, const char *cmd,
if (cmd[1] == 'f')
success = describeFunctions(&cmd[2], pattern,
arg_patterns, num_arg_patterns,
- show_verbose, show_system);
+ verbose, show_system);
else
success = describeOperators(pattern,
arg_patterns, num_arg_patterns,
- show_verbose, show_system);
+ verbose, show_system);
while (--num_arg_patterns >= 0)
free(arg_patterns[num_arg_patterns]);
@@ -2014,9 +2012,9 @@ exec_command_lo(PsqlScanState scan_state, bool active_branch, const char *cmd)
}
else if (strcmp(cmd + 3, "list") == 0)
- success = listLargeObjects(false);
+ success = listLargeObjects(0);
else if (strcmp(cmd + 3, "list+") == 0)
- success = listLargeObjects(true);
+ success = listLargeObjects(1);
else if (strcmp(cmd + 3, "unlink") == 0)
{
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index f7f737e8b59..8ab4b4cb46a 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -33,7 +33,7 @@ static const char *map_typename_pattern(const char *pattern);
static bool describeOneTableDetails(const char *schemaname,
const char *relationname,
const char *oid,
- bool verbose);
+ int verbose);
static void add_tablespace_footer(printTableContent *const cont, char relkind,
Oid tablespace, const bool newline);
static void add_role_attribute(PQExpBuffer buf, const char *const str);
@@ -68,7 +68,7 @@ static bool validateSQLNamePattern(PQExpBuffer buf, const char *pattern,
* Takes an optional regexp to select particular aggregates
*/
bool
-describeAggregates(const char *pattern, bool verbose, bool showSystem)
+describeAggregates(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -169,7 +169,7 @@ describeAccessMethods(const char *pattern, int verbose)
gettext_noop("Table"),
gettext_noop("Type"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBuffer(&buf,
",\n amhandler AS \"%s\",\n"
@@ -235,7 +235,7 @@ describeTablespaces(const char *pattern, int verbose)
gettext_noop("Owner"),
gettext_noop("Location"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n ");
printACLColumn(&buf, "spcacl");
@@ -301,7 +301,7 @@ describeTablespaces(const char *pattern, int verbose)
bool
describeFunctions(const char *functypes, const char *func_pattern,
char **arg_patterns, int num_arg_patterns,
- bool verbose, bool showSystem)
+ int verbose, bool showSystem)
{
bool showAggregate = strchr(functypes, 'a') != NULL;
bool showNormal = strchr(functypes, 'n') != NULL;
@@ -386,7 +386,7 @@ describeFunctions(const char *functypes, const char *func_pattern,
gettext_noop("func"),
gettext_noop("Type"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBuffer(&buf,
",\n CASE\n"
@@ -441,7 +441,7 @@ describeFunctions(const char *functypes, const char *func_pattern,
i, i, i, i, i, i);
}
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_language l ON l.oid = p.prolang\n");
@@ -627,7 +627,7 @@ error_return:
* describe types
*/
bool
-describeTypes(const char *pattern, bool verbose, bool showSystem)
+describeTypes(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -640,7 +640,7 @@ describeTypes(const char *pattern, bool verbose, bool showSystem)
" pg_catalog.format_type(t.oid, NULL) AS \"%s\",\n",
gettext_noop("Schema"),
gettext_noop("Name"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBuffer(&buf,
" t.typname AS \"%s\",\n"
@@ -785,7 +785,7 @@ map_typename_pattern(const char *pattern)
bool
describeOperators(const char *oper_pattern,
char **arg_patterns, int num_arg_patterns,
- bool verbose, bool showSystem)
+ int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -821,7 +821,7 @@ describeOperators(const char *oper_pattern,
gettext_noop("Right arg type"),
gettext_noop("Result type"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
" o.oprcode AS \"%s\",\n",
gettext_noop("Function"));
@@ -1455,10 +1455,10 @@ error_return:
* This routine finds the tables to be displayed, and calls
* describeOneTableDetails for each one.
*
- * verbose: if true, this is \d+
+ * verbose: this is \d+ (or \d++)
*/
bool
-describeTableDetails(const char *pattern, bool verbose, bool showSystem)
+describeTableDetails(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -1544,7 +1544,7 @@ static bool
describeOneTableDetails(const char *schemaname,
const char *relationname,
const char *oid,
- bool verbose)
+ int verbose)
{
bool retval = false;
PQExpBufferData buf;
@@ -1917,7 +1917,7 @@ describeOneTableDetails(const char *schemaname,
" pg_catalog.pg_options_to_table(attfdwoptions)), ', ') || ')' END AS attfdwoptions");
fdwopts_col = cols++;
}
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n a.attstorage");
attstorage_col = cols++;
@@ -2175,7 +2175,7 @@ describeOneTableDetails(const char *schemaname,
"false as inhdetachpending");
/* If verbose, also request the partition constraint definition */
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
",\n pg_catalog.pg_get_partition_constraintdef(c.oid)");
appendPQExpBuffer(&buf,
@@ -2198,7 +2198,7 @@ describeOneTableDetails(const char *schemaname,
strcmp(detached, "t") == 0 ? " DETACH PENDING" : "");
printTableAddFooter(&cont, tmpbuf.data);
- if (verbose)
+ if (verbose > 0)
{
char *partconstraintdef = NULL;
@@ -3434,7 +3434,7 @@ describeOneTableDetails(const char *schemaname,
printfPQExpBuffer(&buf, _("Number of partitions: %d"), tuples);
printTableAddFooter(&cont, buf.data);
}
- else if (!verbose)
+ else if (verbose == 0)
{
/* print the number of child tables, if any */
if (tuples > 0)
@@ -3486,7 +3486,7 @@ describeOneTableDetails(const char *schemaname,
printTableAddFooter(&cont, buf.data);
}
- if (verbose &&
+ if (verbose > 0 &&
(tableinfo.relkind == RELKIND_RELATION ||
tableinfo.relkind == RELKIND_MATVIEW) &&
@@ -3510,7 +3510,7 @@ describeOneTableDetails(const char *schemaname,
}
/* OIDs, if verbose and not a materialized view */
- if (verbose && tableinfo.relkind != RELKIND_MATVIEW && tableinfo.hasoids)
+ if (verbose > 0 && tableinfo.relkind != RELKIND_MATVIEW && tableinfo.hasoids)
printTableAddFooter(&cont, _("Has OIDs: yes"));
/* Tablespace info */
@@ -3518,7 +3518,7 @@ describeOneTableDetails(const char *schemaname,
true);
/* Access method info */
- if (verbose && tableinfo.relam != NULL && !pset.hide_tableam)
+ if (verbose > 0 && tableinfo.relam != NULL && !pset.hide_tableam)
{
printfPQExpBuffer(&buf, _("Access method: %s"), tableinfo.relam);
printTableAddFooter(&cont, buf.data);
@@ -3526,7 +3526,7 @@ describeOneTableDetails(const char *schemaname,
}
/* reloptions, if verbose */
- if (verbose &&
+ if (verbose > 0 &&
tableinfo.reloptions && tableinfo.reloptions[0] != '\0')
{
const char *t = _("Options");
@@ -3626,7 +3626,7 @@ add_tablespace_footer(printTableContent *const cont, char relkind,
* Describes roles. Any schema portion of the pattern is ignored.
*/
bool
-describeRoles(const char *pattern, bool verbose, bool showSystem)
+describeRoles(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -3652,11 +3652,12 @@ describeRoles(const char *pattern, bool verbose, bool showSystem)
" JOIN pg_catalog.pg_roles b ON (m.roleid = b.oid)\n"
" WHERE m.member = r.oid) as memberof");
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, "\n, pg_catalog.shobj_description(r.oid, 'pg_authid') AS description");
ncols++;
}
+
appendPQExpBufferStr(&buf, "\n, r.rolreplication");
if (pset.sversion >= 90500)
@@ -3693,7 +3694,7 @@ describeRoles(const char *pattern, bool verbose, bool showSystem)
/* ignores implicit memberships from superuser & pg_database_owner */
printTableAddHeader(&cont, gettext_noop("Member of"), true, align);
- if (verbose)
+ if (verbose > 0)
printTableAddHeader(&cont, gettext_noop("Description"), true, align);
for (i = 0; i < nrows; i++)
@@ -3752,7 +3753,7 @@ describeRoles(const char *pattern, bool verbose, bool showSystem)
printTableAddCell(&cont, PQgetvalue(res, i, 8), false, false);
- if (verbose)
+ if (verbose > 0)
printTableAddCell(&cont, PQgetvalue(res, i, 9), false, false);
}
termPQExpBuffer(&buf);
@@ -3862,7 +3863,7 @@ error_return:
* (any order of the above is fine)
*/
bool
-listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSystem)
+listTables(const char *tabtypes, const char *pattern, int verbose, bool showSystem)
{
bool showTables = strchr(tabtypes, 't') != NULL;
bool showIndexes = strchr(tabtypes, 'i') != NULL;
@@ -3921,7 +3922,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys
cols_so_far++;
}
- if (verbose)
+ if (verbose > 0)
{
/*
* Show whether a relation is permanent, temporary, or unlogged.
@@ -4061,7 +4062,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys
* and you can mix and match these in any order.
*/
bool
-listPartitionedTables(const char *reltypes, const char *pattern, bool verbose)
+listPartitionedTables(const char *reltypes, const char *pattern, int verbose)
{
bool showTables = strchr(reltypes, 't') != NULL;
bool showIndexes = strchr(reltypes, 'i') != NULL;
@@ -4136,7 +4137,7 @@ listPartitionedTables(const char *reltypes, const char *pattern, bool verbose)
",\n c2.oid::pg_catalog.regclass as \"%s\"",
gettext_noop("Table"));
- if (verbose)
+ if (verbose > 0)
{
if (showNested)
{
@@ -4171,7 +4172,7 @@ listPartitionedTables(const char *reltypes, const char *pattern, bool verbose)
appendPQExpBufferStr(&buf,
"\n LEFT JOIN pg_catalog.pg_inherits inh ON c.oid = inh.inhrelid");
- if (verbose)
+ if (verbose > 0)
{
if (pset.sversion < 120000)
{
@@ -4262,7 +4263,7 @@ listPartitionedTables(const char *reltypes, const char *pattern, bool verbose)
* Describes languages.
*/
bool
-listLanguages(const char *pattern, bool verbose, bool showSystem)
+listLanguages(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -4278,7 +4279,7 @@ listLanguages(const char *pattern, bool verbose, bool showSystem)
gettext_noop("Owner"),
gettext_noop("Trusted"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBuffer(&buf,
",\n NOT l.lanispl AS \"%s\",\n"
@@ -4339,7 +4340,7 @@ listLanguages(const char *pattern, bool verbose, bool showSystem)
* Describes domains.
*/
bool
-listDomains(const char *pattern, bool verbose, bool showSystem)
+listDomains(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -4366,7 +4367,7 @@ listDomains(const char *pattern, bool verbose, bool showSystem)
gettext_noop("Default"),
gettext_noop("Check"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n ");
printACLColumn(&buf, "t.typacl");
@@ -4379,7 +4380,7 @@ listDomains(const char *pattern, bool verbose, bool showSystem)
"\nFROM pg_catalog.pg_type t\n"
" LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_description d "
"ON d.classoid = t.tableoid AND d.objoid = t.oid "
@@ -4423,7 +4424,7 @@ listDomains(const char *pattern, bool verbose, bool showSystem)
* Describes conversions.
*/
bool
-listConversions(const char *pattern, bool verbose, bool showSystem)
+listConversions(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -4447,7 +4448,7 @@ listConversions(const char *pattern, bool verbose, bool showSystem)
gettext_noop("yes"), gettext_noop("no"),
gettext_noop("Default?"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n d.description AS \"%s\"",
gettext_noop("Description"));
@@ -4457,7 +4458,7 @@ listConversions(const char *pattern, bool verbose, bool showSystem)
" JOIN pg_catalog.pg_namespace n "
"ON n.oid = c.connamespace\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
"LEFT JOIN pg_catalog.pg_description d "
"ON d.classoid = c.tableoid\n"
@@ -4504,7 +4505,7 @@ listConversions(const char *pattern, bool verbose, bool showSystem)
* Describes configuration parameters.
*/
bool
-describeConfigurationParameters(const char *pattern, bool verbose,
+describeConfigurationParameters(const char *pattern, int verbose,
bool showSystem)
{
PQExpBufferData buf;
@@ -4573,7 +4574,7 @@ describeConfigurationParameters(const char *pattern, bool verbose,
* Describes Event Triggers.
*/
bool
-listEventTriggers(const char *pattern, bool verbose)
+listEventTriggers(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -4614,7 +4615,7 @@ listEventTriggers(const char *pattern, bool verbose)
gettext_noop("Enabled"),
gettext_noop("Function"),
gettext_noop("Tags"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\npg_catalog.obj_description(e.oid, 'pg_event_trigger') as \"%s\"",
gettext_noop("Description"));
@@ -4751,7 +4752,7 @@ listExtendedStats(const char *pattern)
* Describes casts.
*/
bool
-listCasts(const char *pattern, bool verbose)
+listCasts(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -4792,7 +4793,7 @@ listCasts(const char *pattern, bool verbose)
gettext_noop("yes"),
gettext_noop("Implicit?"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n d.description AS \"%s\"",
gettext_noop("Description"));
@@ -4813,7 +4814,7 @@ listCasts(const char *pattern, bool verbose)
" LEFT JOIN pg_catalog.pg_namespace nt\n"
" ON nt.oid = tt.typnamespace\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_description d\n"
" ON d.classoid = c.tableoid AND d.objoid = "
@@ -4870,7 +4871,7 @@ error_return:
* Describes collations.
*/
bool
-listCollations(const char *pattern, bool verbose, bool showSystem)
+listCollations(const char *pattern, int verbose, bool showSystem)
{
PQExpBufferData buf;
PGresult *res;
@@ -4930,7 +4931,7 @@ listCollations(const char *pattern, bool verbose, bool showSystem)
gettext_noop("yes"),
gettext_noop("Deterministic?"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n pg_catalog.obj_description(c.oid, 'pg_collation') AS \"%s\"",
gettext_noop("Description"));
@@ -5000,7 +5001,7 @@ listSchemas(const char *pattern, int verbose, bool showSystem)
gettext_noop("Name"),
gettext_noop("Owner"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n ");
printACLColumn(&buf, "n.nspacl");
@@ -5112,13 +5113,13 @@ error_return:
* list text search parsers
*/
bool
-listTSParsers(const char *pattern, bool verbose)
+listTSParsers(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
printQueryOpt myopt = pset.popt;
- if (verbose)
+ if (verbose > 0)
return listTSParsersVerbose(pattern);
initPQExpBuffer(&buf);
@@ -5362,7 +5363,7 @@ describeOneTSParser(const char *oid, const char *nspname, const char *prsname)
* list text search dictionaries
*/
bool
-listTSDictionaries(const char *pattern, bool verbose)
+listTSDictionaries(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5377,7 +5378,7 @@ listTSDictionaries(const char *pattern, bool verbose)
gettext_noop("Schema"),
gettext_noop("Name"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBuffer(&buf,
" ( SELECT COALESCE(nt.nspname, '(null)')::pg_catalog.text || '.' || t.tmplname FROM\n"
@@ -5428,7 +5429,7 @@ listTSDictionaries(const char *pattern, bool verbose)
* list text search templates
*/
bool
-listTSTemplates(const char *pattern, bool verbose)
+listTSTemplates(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5436,7 +5437,7 @@ listTSTemplates(const char *pattern, bool verbose)
initPQExpBuffer(&buf);
- if (verbose)
+ if (verbose > 0)
printfPQExpBuffer(&buf,
"SELECT\n"
" n.nspname AS \"%s\",\n"
@@ -5494,13 +5495,13 @@ listTSTemplates(const char *pattern, bool verbose)
* list text search configurations
*/
bool
-listTSConfigs(const char *pattern, bool verbose)
+listTSConfigs(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
printQueryOpt myopt = pset.popt;
- if (verbose)
+ if (verbose > 0)
return listTSConfigsVerbose(pattern);
initPQExpBuffer(&buf);
@@ -5700,7 +5701,7 @@ describeOneTSConfig(const char *oid, const char *nspname, const char *cfgname,
* Describes foreign-data wrappers
*/
bool
-listForeignDataWrappers(const char *pattern, bool verbose)
+listForeignDataWrappers(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5717,7 +5718,7 @@ listForeignDataWrappers(const char *pattern, bool verbose)
gettext_noop("Handler"),
gettext_noop("Validator"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n ");
printACLColumn(&buf, "fdwacl");
@@ -5735,7 +5736,7 @@ listForeignDataWrappers(const char *pattern, bool verbose)
appendPQExpBufferStr(&buf, "\nFROM pg_catalog.pg_foreign_data_wrapper fdw\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
"LEFT JOIN pg_catalog.pg_description d\n"
" ON d.classoid = fdw.tableoid "
@@ -5772,7 +5773,7 @@ listForeignDataWrappers(const char *pattern, bool verbose)
* Describes foreign servers.
*/
bool
-listForeignServers(const char *pattern, bool verbose)
+listForeignServers(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5787,7 +5788,7 @@ listForeignServers(const char *pattern, bool verbose)
gettext_noop("Owner"),
gettext_noop("Foreign-data wrapper"));
- if (verbose)
+ if (verbose > 0)
{
appendPQExpBufferStr(&buf, ",\n ");
printACLColumn(&buf, "s.srvacl");
@@ -5812,7 +5813,7 @@ listForeignServers(const char *pattern, bool verbose)
"\nFROM pg_catalog.pg_foreign_server s\n"
" JOIN pg_catalog.pg_foreign_data_wrapper f ON f.oid=s.srvfdw\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
"LEFT JOIN pg_catalog.pg_description d\n "
"ON d.classoid = s.tableoid AND d.objoid = s.oid "
@@ -5849,7 +5850,7 @@ listForeignServers(const char *pattern, bool verbose)
* Describes user mappings.
*/
bool
-listUserMappings(const char *pattern, bool verbose)
+listUserMappings(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5862,7 +5863,7 @@ listUserMappings(const char *pattern, bool verbose)
gettext_noop("Server"),
gettext_noop("User name"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n CASE WHEN umoptions IS NULL THEN '' ELSE "
" '(' || pg_catalog.array_to_string(ARRAY(SELECT "
@@ -5905,7 +5906,7 @@ listUserMappings(const char *pattern, bool verbose)
* Describes foreign tables.
*/
bool
-listForeignTables(const char *pattern, bool verbose)
+listForeignTables(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -5920,7 +5921,7 @@ listForeignTables(const char *pattern, bool verbose)
gettext_noop("Table"),
gettext_noop("Server"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n CASE WHEN ftoptions IS NULL THEN '' ELSE "
" '(' || pg_catalog.array_to_string(ARRAY(SELECT "
@@ -5940,7 +5941,7 @@ listForeignTables(const char *pattern, bool verbose)
" ON n.oid = c.relnamespace\n"
" INNER JOIN pg_catalog.pg_foreign_server s"
" ON s.oid = ft.ftserver\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_description d\n"
" ON d.classoid = c.tableoid AND "
@@ -6504,7 +6505,7 @@ error_return:
* Takes an optional regexp to select particular subscriptions
*/
bool
-describeSubscriptions(const char *pattern, bool verbose)
+describeSubscriptions(const char *pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -6534,7 +6535,7 @@ describeSubscriptions(const char *pattern, bool verbose)
gettext_noop("Enabled"),
gettext_noop("Publication"));
- if (verbose)
+ if (verbose > 0)
{
/* Binary mode and streaming are only supported in v14 and higher */
if (pset.sversion >= 140000)
@@ -6645,7 +6646,7 @@ printACLColumn(PQExpBuffer buf, const char *colname)
*/
bool
listOperatorClasses(const char *access_method_pattern,
- const char *type_pattern, bool verbose)
+ const char *type_pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -6680,7 +6681,7 @@ listOperatorClasses(const char *access_method_pattern,
gettext_noop("yes"),
gettext_noop("no"),
gettext_noop("Default?"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n CASE\n"
" WHEN pg_catalog.pg_opfamily_is_visible(of.oid)\n"
@@ -6696,7 +6697,7 @@ listOperatorClasses(const char *access_method_pattern,
" LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.opcnamespace\n"
" LEFT JOIN pg_catalog.pg_type t ON t.oid = c.opcintype\n"
" LEFT JOIN pg_catalog.pg_namespace tn ON tn.oid = t.typnamespace\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = c.opcfamily\n"
" LEFT JOIN pg_catalog.pg_namespace ofn ON ofn.oid = of.opfnamespace\n");
@@ -6747,7 +6748,7 @@ error_return:
*/
bool
listOperatorFamilies(const char *access_method_pattern,
- const char *type_pattern, bool verbose)
+ const char *type_pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -6772,7 +6773,7 @@ listOperatorFamilies(const char *access_method_pattern,
gettext_noop("AM"),
gettext_noop("Operator family"),
gettext_noop("Applicable types"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
",\n pg_catalog.pg_get_userbyid(f.opfowner) AS \"%s\"\n",
gettext_noop("Owner"));
@@ -6837,7 +6838,7 @@ error_return:
*/
bool
listOpFamilyOperators(const char *access_method_pattern,
- const char *family_pattern, bool verbose)
+ const char *family_pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -6870,7 +6871,7 @@ listOpFamilyOperators(const char *access_method_pattern,
gettext_noop("search"),
gettext_noop("Purpose"));
- if (verbose)
+ if (verbose > 0)
appendPQExpBuffer(&buf,
", ofs.opfname AS \"%s\"\n",
gettext_noop("Sort opfamily"));
@@ -6879,7 +6880,7 @@ listOpFamilyOperators(const char *access_method_pattern,
" LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = o.amopfamily\n"
" LEFT JOIN pg_catalog.pg_am am ON am.oid = of.opfmethod AND am.oid = o.amopmethod\n"
" LEFT JOIN pg_catalog.pg_namespace nsf ON of.opfnamespace = nsf.oid\n");
- if (verbose)
+ if (verbose > 0)
appendPQExpBufferStr(&buf,
" LEFT JOIN pg_catalog.pg_opfamily ofs ON ofs.oid = o.amopsortfamily\n");
@@ -6936,7 +6937,7 @@ error_return:
*/
bool
listOpFamilyFunctions(const char *access_method_pattern,
- const char *family_pattern, bool verbose)
+ const char *family_pattern, int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -6963,7 +6964,7 @@ listOpFamilyFunctions(const char *access_method_pattern,
gettext_noop("Registered right type"),
gettext_noop("Number"));
- if (!verbose)
+ if (verbose == 0)
appendPQExpBuffer(&buf,
", p.proname AS \"%s\"\n",
gettext_noop("Function"));
@@ -7025,7 +7026,7 @@ error_return:
* Lists large objects
*/
bool
-listLargeObjects(bool verbose)
+listLargeObjects(int verbose)
{
PQExpBufferData buf;
PGresult *res;
@@ -7039,7 +7040,7 @@ listLargeObjects(bool verbose)
gettext_noop("ID"),
gettext_noop("Owner"));
- if (verbose)
+ if (verbose > 0)
{
printACLColumn(&buf, "lomacl");
appendPQExpBufferStr(&buf, ",\n ");
diff --git a/src/bin/psql/describe.h b/src/bin/psql/describe.h
index d2fd8a72a36..23d601fbc97 100644
--- a/src/bin/psql/describe.h
+++ b/src/bin/psql/describe.h
@@ -10,7 +10,7 @@
/* \da */
-extern bool describeAggregates(const char *pattern, bool verbose, bool showSystem);
+extern bool describeAggregates(const char *pattern, int verbose, bool showSystem);
/* \dA */
extern bool describeAccessMethods(const char *pattern, int verbose);
@@ -21,18 +21,18 @@ extern bool describeTablespaces(const char *pattern, int verbose);
/* \df, \dfa, \dfn, \dft, \dfw, etc. */
extern bool describeFunctions(const char *functypes, const char *func_pattern,
char **arg_patterns, int num_arg_patterns,
- bool verbose, bool showSystem);
+ int verbose, bool showSystem);
/* \dT */
-extern bool describeTypes(const char *pattern, bool verbose, bool showSystem);
+extern bool describeTypes(const char *pattern, int verbose, bool showSystem);
/* \do */
extern bool describeOperators(const char *oper_pattern,
char **arg_patterns, int num_arg_patterns,
- bool verbose, bool showSystem);
+ int verbose, bool showSystem);
/* \du, \dg */
-extern bool describeRoles(const char *pattern, bool verbose, bool showSystem);
+extern bool describeRoles(const char *pattern, int verbose, bool showSystem);
/* \drds */
extern bool listDbRoleSettings(const char *pattern, const char *pattern2);
@@ -47,62 +47,62 @@ extern bool listDefaultACLs(const char *pattern);
extern bool objectDescription(const char *pattern, bool showSystem);
/* \d foo */
-extern bool describeTableDetails(const char *pattern, bool verbose, bool showSystem);
+extern bool describeTableDetails(const char *pattern, int verbose, bool showSystem);
/* \dF */
-extern bool listTSConfigs(const char *pattern, bool verbose);
+extern bool listTSConfigs(const char *pattern, int verbose);
/* \dFp */
-extern bool listTSParsers(const char *pattern, bool verbose);
+extern bool listTSParsers(const char *pattern, int verbose);
/* \dFd */
-extern bool listTSDictionaries(const char *pattern, bool verbose);
+extern bool listTSDictionaries(const char *pattern, int verbose);
/* \dFt */
-extern bool listTSTemplates(const char *pattern, bool verbose);
+extern bool listTSTemplates(const char *pattern, int verbose);
/* \l */
extern bool listAllDbs(const char *pattern, int verbose);
/* \dt, \di, \ds, \dS, etc. */
-extern bool listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSystem);
+extern bool listTables(const char *tabtypes, const char *pattern, int verbose, bool showSystem);
/* \dP */
-extern bool listPartitionedTables(const char *reltypes, const char *pattern, bool verbose);
+extern bool listPartitionedTables(const char *reltypes, const char *pattern, int verbose);
/* \dD */
-extern bool listDomains(const char *pattern, bool verbose, bool showSystem);
+extern bool listDomains(const char *pattern, int verbose, bool showSystem);
/* \dc */
-extern bool listConversions(const char *pattern, bool verbose, bool showSystem);
+extern bool listConversions(const char *pattern, int verbose, bool showSystem);
/* \dconfig */
-extern bool describeConfigurationParameters(const char *pattern, bool verbose,
+extern bool describeConfigurationParameters(const char *pattern, int verbose,
bool showSystem);
/* \dC */
-extern bool listCasts(const char *pattern, bool verbose);
+extern bool listCasts(const char *pattern, int verbose);
/* \dO */
-extern bool listCollations(const char *pattern, bool verbose, bool showSystem);
+extern bool listCollations(const char *pattern, int verbose, bool showSystem);
/* \dn */
extern bool listSchemas(const char *pattern, int verbose, bool showSystem);
/* \dew */
-extern bool listForeignDataWrappers(const char *pattern, bool verbose);
+extern bool listForeignDataWrappers(const char *pattern, int verbose);
/* \des */
-extern bool listForeignServers(const char *pattern, bool verbose);
+extern bool listForeignServers(const char *pattern, int verbose);
/* \deu */
-extern bool listUserMappings(const char *pattern, bool verbose);
+extern bool listUserMappings(const char *pattern, int verbose);
/* \det */
-extern bool listForeignTables(const char *pattern, bool verbose);
+extern bool listForeignTables(const char *pattern, int verbose);
/* \dL */
-extern bool listLanguages(const char *pattern, bool verbose, bool showSystem);
+extern bool listLanguages(const char *pattern, int verbose, bool showSystem);
/* \dx */
extern bool listExtensions(const char *pattern);
@@ -114,7 +114,7 @@ extern bool listExtensionContents(const char *pattern);
extern bool listExtendedStats(const char *pattern);
/* \dy */
-extern bool listEventTriggers(const char *pattern, bool verbose);
+extern bool listEventTriggers(const char *pattern, int verbose);
/* \dRp */
bool listPublications(const char *pattern);
@@ -123,27 +123,27 @@ bool listPublications(const char *pattern);
bool describePublications(const char *pattern);
/* \dRs */
-bool describeSubscriptions(const char *pattern, bool verbose);
+bool describeSubscriptions(const char *pattern, int verbose);
/* \dAc */
extern bool listOperatorClasses(const char *access_method_pattern,
const char *type_pattern,
- bool verbose);
+ int verbose);
/* \dAf */
extern bool listOperatorFamilies(const char *access_method_pattern,
const char *type_pattern,
- bool verbose);
+ int verbose);
/* \dAo */
extern bool listOpFamilyOperators(const char *access_method_pattern,
- const char *family_pattern, bool verbose);
+ const char *family_pattern, int verbose);
/* \dAp */
extern bool listOpFamilyFunctions(const char *access_method_pattern,
- const char *family_pattern, bool verbose);
+ const char *family_pattern, int verbose);
/* \dl or \lo_list */
-extern bool listLargeObjects(bool verbose);
+extern bool listLargeObjects(int verbose);
#endif /* DESCRIBE_H */
--
2.34.1
--mDjj41GS8UJ6l/Dp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0004-Move-the-double-plus-Size-columns-to-the-right.patch"
^ permalink raw reply [nested|flat] 99+ messages in thread
* Data is copied twice when specifying both child and parent table in publication
@ 2021-10-15 11:22 [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2021-10-15 11:22 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>
Hi,
In another logical replication related thread[1], my colleague Greg found that
if publish_via_partition_root is true, then the child table's data will be
copied twice when adding both child and parent table to the publication.
Example:
-----
Pub:
create table tbl1 (a int) partition by range (a);
create table tbl1_part1 partition of tbl1 for values from (1) to (10);
create table tbl1_part2 partition of tbl1 for values from (10) to (20);
create publication pub for table tbl1, tbl1_part1 with (publish_via_partition_root=on);
insert into tbl1_part1 values(1);
Sub:
create table tbl1 (a int) partition by range (a);
create table tbl1_part1 partition of tbl1 for values from (1) to (10);
create table tbl1_part2 partition of tbl1 for values from (10) to (20);
create subscription sub CONNECTION 'dbname=postgres port=10000' publication pub;
-- data is copied twice
select * from tbl1_part1;
a
---
1
1
-----
The reason is that the subscriber will fetch the table list from publisher
using the following sql[2] and the subscriber will execute table
synchronization for each table in the query results in this case. But
tbl1_part1 is a partition of tbl1, so the data of tbl1_part1 was copied twice.
[2]
select * from pg_publication_tables;
pubname | schemaname | tablename
---------+------------+------------
pub | public | tbl1
pub | public | tbl1_part1
IMO, it looks like a bug and it's more natural to only execute the table
synchronization for the parent table in the above case. Because as the document
said: if publish_via_partition_root is true, "changes in a partitioned table
(or on its partitions) contained in the publication will be published using the
identity and schema of the partitioned table rather than that of the individual
partitions that are actually changed;"
To fix it, I think we should fix function GetPublicationRelations which
generate data for the view pg_publication_tables and make it only show the
parent table if publish_via_partition_root is true. And for other future
feature like schema level publication, we can also follow this to exclude
partitions if their parent is specified by FOR TABLE in the same publication.
Attach a patch to fix it.
Thoughts ?
[1] https://www.postgresql.org/message-id/CAJcOf-eBhDUT2J5zs8Z0qEMiZUdhinX%2BbuGX3GN4V83fPnZV3Q%40mail.g...
Best regards,
Hou zhijie
Attachments:
[application/octet-stream] 0001-fix-double-publish.patch (7.2K, ../../OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-0001-fix-double-publish.patch)
download | inline diff:
From af38de251ac5fb321f19273ec079eac3f08f4f08 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 15 Oct 2021 18:23:30 +0800
Subject: [PATCH] fix double publish
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 51 +++++++++++++++++++++--
src/backend/commands/publicationcmds.c | 8 ++--
src/include/catalog/pg_publication.h | 3 +-
src/test/regress/expected/publication.out | 9 ++++
src/test/regress/sql/publication.sql | 4 ++
5 files changed, 67 insertions(+), 8 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 9cd0c82f93..8f039599ee 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -291,11 +291,18 @@ GetRelationPublications(Oid relid)
*
* This should only be used FOR TABLE publications, the FOR ALL TABLES
* should use GetAllTablesPublicationRelations().
+ *
+ * If pubviaroot is true, we must exclude partitions in favor of including the
+ * root partitioned tables.
*/
List *
-GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt)
+GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt,
+ bool pubviaroot)
{
List *result;
+ List *relids;
+ ListCell *lc;
+ ListCell *lc2;
Relation pubrelsrel;
ScanKeyData scankey;
SysScanDesc scan;
@@ -312,19 +319,54 @@ GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt)
scan = systable_beginscan(pubrelsrel, PublicationRelPrrelidPrpubidIndexId,
true, NULL, 1, &scankey);
- result = NIL;
+ result = relids = NIL;
+
while (HeapTupleIsValid(tup = systable_getnext(scan)))
{
Form_pg_publication_rel pubrel;
pubrel = (Form_pg_publication_rel) GETSTRUCT(tup);
- result = GetPubPartitionOptionRelations(result, pub_partopt,
+ relids = GetPubPartitionOptionRelations(relids, pub_partopt,
pubrel->prrelid);
}
systable_endscan(scan);
table_close(pubrelsrel, AccessShareLock);
+ if (!pubviaroot)
+ return relids;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ /*
+ * Filter out the partitions whose parent tables was also specified in
+ * the publication.
+ */
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ /*
+ * Check if the parent table exists in the published table list or
+ * the schema of the parent table was published in this
+ * publication.
+ */
+ if (list_member_oid(relids, lfirst_oid(lc2)))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
return result;
}
@@ -560,7 +602,8 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
+ PUBLICATION_PART_LEAF,
+ publication->pubviaroot);
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c
index 9c7f91611d..836b6cf421 100644
--- a/src/backend/commands/publicationcmds.c
+++ b/src/backend/commands/publicationcmds.c
@@ -324,7 +324,8 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt,
* trees, not just those explicitly mentioned in the publication.
*/
List *relids = GetPublicationRelations(pubform->oid,
- PUBLICATION_PART_ALL);
+ PUBLICATION_PART_ALL,
+ false);
InvalidatePublicationRels(relids);
}
@@ -387,7 +388,8 @@ AlterPublicationTables(AlterPublicationStmt *stmt, Relation rel,
else /* DEFELEM_SET */
{
List *oldrelids = GetPublicationRelations(pubid,
- PUBLICATION_PART_ROOT);
+ PUBLICATION_PART_ROOT,
+ false);
List *delrels = NIL;
ListCell *oldlc;
@@ -538,7 +540,7 @@ RemovePublicationById(Oid pubid)
if (!HeapTupleIsValid(tup))
elog(ERROR, "cache lookup failed for publication %u", pubid);
- pubform = (Form_pg_publication)GETSTRUCT(tup);
+ pubform = (Form_pg_publication) GETSTRUCT(tup);
/* Invalidate relcache so that publication info is rebuilt. */
if (pubform->puballtables)
diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h
index 82f2536c65..721233b226 100644
--- a/src/include/catalog/pg_publication.h
+++ b/src/include/catalog/pg_publication.h
@@ -108,7 +108,8 @@ typedef enum PublicationPartOpt
PUBLICATION_PART_ALL,
} PublicationPartOpt;
-extern List *GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt);
+extern List *GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt,
+ bool pubviaroot);
extern List *GetAllTablesPublications(void);
extern List *GetAllTablesPublicationRelations(bool pubviaroot);
extern List *GetPubPartitionOptionRelations(List *result,
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 82bce9be09..5f5445a985 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -165,6 +165,15 @@ HINT: To enable updating the table, set REPLICA IDENTITY using ALTER TABLE.
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
+ pubname | schemaname | tablename
+-------------------+------------+----------------
+ testpub_forparted | public | testpub_parted
+(1 row)
+
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
-- Test cache invalidation FOR ALL TABLES publication
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d575b..d3f2b2fa11 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-16 06:30 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: [email protected] @ 2021-10-16 06:30 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>
On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> Attach a patch to fix it.
Attach a new version patch which refactor the fix code in a cleaner way.
Best regards,
Hou zj
Attachments:
[application/octet-stream] v2-0001-fix-double-publish.patch (5.9K, ../../OS0PR01MB57166BB84617D22872B2ECC994BA9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v2-0001-fix-double-publish.patch)
download | inline diff:
From c0cbecb941e4f659142d65b7eac2df801a9b0fa7 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Sat, 16 Oct 2021 14:14:40 +0800
Subject: [PATCH] fix double publish
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 77 +++++++++++++++++++++++++++++--
src/include/catalog/pg_publication.h | 1 +
src/test/regress/expected/publication.out | 9 ++++
src/test/regress/sql/publication.sql | 4 ++
4 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 9cd0c82..7c9208b 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables was also specified in
+ * the publication.
+ */
+static List *
+filter_out_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ /*
+ * Check if the parent table exists in the published table list.
+ */
+ if (list_member_oid(relids, lfirst_oid(lc2)))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -328,6 +367,32 @@ GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt)
return result;
}
+ /*
+ * Gets list of relation explicitly mentioned in the publication.
+ *
+ * This should only be used FOR TABLE publications, the FOR ALL TABLES
+ * should use GetAllTablesPublicationRelations().
+ *
+ * If pubviaroot is true, we must exclude partitions in favor of including the
+ * root partitioned tables.
+ */
+List *
+GetPublicationRelationsExtended(Oid pubid, bool pubviaroot)
+{
+ List *result;
+
+ result = GetPublicationRelations(pubid, PUBLICATION_PART_ROOT);
+
+ /*
+ * Filter out the partitions whose parent table is also in the same
+ * publication.
+ */
+ if (pubviaroot)
+ return filter_out_partitions(result);
+ else
+ return result;
+}
+
/*
* Gets list of publication oids for publications marked as FOR ALL TABLES.
*/
@@ -557,10 +622,14 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
- tables = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
+ {
+ if (publication->pubviaroot)
+ tables = GetPublicationRelationsExtended(publication->oid,
+ publication->pubviaroot);
+ else
+ tables = GetPublicationRelations(publication->oid,
+ PUBLICATION_PART_LEAF);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h
index 82f2536..d5e3c02 100644
--- a/src/include/catalog/pg_publication.h
+++ b/src/include/catalog/pg_publication.h
@@ -109,6 +109,7 @@ typedef enum PublicationPartOpt
} PublicationPartOpt;
extern List *GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt);
+extern List *GetPublicationRelationsExtended(Oid pubid, bool pubviaroot);
extern List *GetAllTablesPublications(void);
extern List *GetAllTablesPublicationRelations(bool pubviaroot);
extern List *GetPubPartitionOptionRelations(List *result,
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 82bce9b..5f5445a 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -165,6 +165,15 @@ HINT: To enable updating the table, set REPLICA IDENTITY using ALTER TABLE.
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
+ pubname | schemaname | tablename
+-------------------+------------+----------------
+ testpub_forparted | public | testpub_parted
+(1 row)
+
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
-- Test cache invalidation FOR ALL TABLES publication
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d5..d3f2b2f 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-18 02:57 Greg Nancarrow <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-18 02:57 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>
On Sat, Oct 16, 2021 at 5:30 PM [email protected]
<[email protected]> wrote:
>
> On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> > Attach a patch to fix it.
> Attach a new version patch which refactor the fix code in a cleaner way.
>
I have not debugged it yet to find out why, but with the patch
applied, the original double-publish problem that I reported
(converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
occurs.
The steps are below:
CREATE SCHEMA sch;
CREATE SCHEMA sch1;
CREATE TABLE sch.sale (sale_date date not null, country_code text,
product_sku text, units integer) PARTITION BY RANGE (sale_date);
CREATE TABLE sch1.sale_201901 PARTITION OF sch.sale FOR VALUES FROM
('2019-01-01') TO ('2019-02-01');
CREATE TABLE sch1.sale_201902 PARTITION OF sch.sale FOR VALUES FROM
('2019-02-01') TO ('2019-03-01');
(1) PUB: CREATE PUBLICATION pub FOR TABLE sch1.sale_201901,
sch1.sale_201902 WITH (publish_via_partition_root=true);
(2) SUB: CREATE SUBSCRIPTION sub CONNECTION 'dbname=postgres
host=localhost port=5432' PUBLICATION pub;
(3) PUB: INSERT INTO sch.sale VALUES('2019-01-01', 'AU', 'cpu', 5),
('2019-01-02', 'AU', 'disk', 8);
(4) SUB: SELECT * FROM sch.sale;
(5) PUB: ALTER PUBLICATION pub ADD TABLE sch.sale;
(6) SUB: ALTER SUBSCRIPTION sub REFRESH PUBLICATION;
(7) SUB: SELECT * FROM sch.sale;
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-18 06:00 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: Amit Kapila @ 2021-10-18 06:00 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>
On Mon, Oct 18, 2021 at 8:27 AM Greg Nancarrow <[email protected]> wrote:
>
> On Sat, Oct 16, 2021 at 5:30 PM [email protected]
> <[email protected]> wrote:
> >
> > On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> > > Attach a patch to fix it.
> > Attach a new version patch which refactor the fix code in a cleaner way.
> >
>
> I have not debugged it yet to find out why, but with the patch
> applied, the original double-publish problem that I reported
> (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> occurs.
>
Yeah, I think this is a variant of the problem being fixed by
Hou-San's patch. I think one possible idea to investigate is that on
the subscriber-side, after fetching tables, we check the already
subscribed tables and if the child tables already exist then we ignore
the parent table and vice versa. We might want to consider the case
where a user has toggled the "publish_via_partition_root" parameter.
It seems both these behaviours/problems exist since commit 17b9e7f9
(Support adding partitioned tables to publication). Adding Amit L and
Peter E (people involved in this work) to know their opinion?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-18 08:46 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: [email protected] @ 2021-10-18 08:46 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; [email protected] <[email protected]>
On Saturday, October 16, 2021 2:31 PM [email protected] wrote:
> On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> > Attach a patch to fix it.
> Attach a new version patch which refactor the fix code in a cleaner way.
Although the discussion about the partition behavior[1] is going on,
attach a refactored fix patch which make the pg_publication_tables view be
consistent for FOR TABLE and FOR ALL TABLES here in case someone want
to have a look.
[1] https://www.postgresql.org/message-id/CAA4eK1JC5sy5M_UVoGdgubHN2--peYqApOJkT%3DFLCq%2BVUxqerQ%40mail...
Best regards,
Hou zj
Attachments:
[application/octet-stream] v3-0001-fix-double-publish.patch (4.6K, ../../OS0PR01MB5716EABF947737006B1CA8C394BC9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v3-0001-fix-double-publish.patch)
download | inline diff:
From 30c911c6e6223cf4f9d2141361b8164886857b18 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Mon, 18 Oct 2021 11:46:44 +0800
Subject: [PATCH] fix double publish
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 52 +++++++++++++++++++++++++++++++
src/test/regress/expected/publication.out | 9 ++++++
src/test/regress/sql/publication.sql | 4 +++
3 files changed, 65 insertions(+)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 9cd0c82..bd0db8c 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables was also specified in
+ * the publication.
+ */
+static List *
+filter_out_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ /*
+ * Check if the parent table exists in the published table list.
+ */
+ if (list_member_oid(relids, lfirst_oid(lc2)))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -557,10 +596,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
+ {
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables which
+ * could cause the data of child table double-published in
+ * subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_out_partitions(tables);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 82bce9b..5f5445a 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -165,6 +165,15 @@ HINT: To enable updating the table, set REPLICA IDENTITY using ALTER TABLE.
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
+ pubname | schemaname | tablename
+-------------------+------------+----------------
+ testpub_forparted | public | testpub_parted
+(1 row)
+
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
-- Test cache invalidation FOR ALL TABLES publication
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d5..d3f2b2f 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- add both child and parent table to the publication
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only show parent table when publish_via_partition_root is set
+select * from pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-18 09:02 Amit Langote <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 2 replies; 99+ messages in thread
From: Amit Langote @ 2021-10-18 09:02 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Mon, Oct 18, 2021 at 3:00 PM Amit Kapila <[email protected]> wrote:
> On Mon, Oct 18, 2021 at 8:27 AM Greg Nancarrow <[email protected]> wrote:
> >
> > On Sat, Oct 16, 2021 at 5:30 PM [email protected]
> > <[email protected]> wrote:
> > >
> > > On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> > > > Attach a patch to fix it.
> > > Attach a new version patch which refactor the fix code in a cleaner way.
> > >
> >
> > I have not debugged it yet to find out why, but with the patch
> > applied, the original double-publish problem that I reported
> > (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> > occurs.
> >
>
> Yeah, I think this is a variant of the problem being fixed by
> Hou-San's patch. I think one possible idea to investigate is that on
> the subscriber-side, after fetching tables, we check the already
> subscribed tables and if the child tables already exist then we ignore
> the parent table and vice versa. We might want to consider the case
> where a user has toggled the "publish_via_partition_root" parameter.
>
> It seems both these behaviours/problems exist since commit 17b9e7f9
> (Support adding partitioned tables to publication). Adding Amit L and
> Peter E (people involved in this work) to know their opinion?
I can imagine that the behavior seen here may look surprising, but not
sure if I would call it a bug as such. I do remember thinking about
this case and the current behavior is how I may have coded it to be.
Looking at this command in Hou-san's email:
create publication pub for table tbl1, tbl1_part1 with
(publish_via_partition_root=on);
It's adding both the root partitioned table and the leaf partition
*explicitly*, and it's not clear to me if the latter's inclusion in
the publication should be assumed because the former is found to have
been added to the publication, that is, as far as the latter's
visibility to the subscriber is concerned. It's not a stretch to
imagine that a user may write the command this way to account for a
subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
I don't think we assume anything on the publisher side regarding the
state/configuration of tables on the subscriber side, at least with
publication commands where tables are added to a publication
explicitly, so it is up to the user to make sure that the tables are
not added duplicatively. One may however argue that the way we've
decided to handle FOR ALL TABLES does assume something about
partitions where it skips advertising them to subscribers when
publish_via_partition_root flag is set to true, but that is exactly to
avoid the duplication of data that goes to a subscriber.
--
Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-18 09:28 Amit Kapila <[email protected]>
parent: Amit Langote <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-10-18 09:28 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Mon, Oct 18, 2021 at 2:32 PM Amit Langote <[email protected]> wrote:
>
> On Mon, Oct 18, 2021 at 3:00 PM Amit Kapila <[email protected]> wrote:
> > On Mon, Oct 18, 2021 at 8:27 AM Greg Nancarrow <[email protected]> wrote:
> > >
> > > On Sat, Oct 16, 2021 at 5:30 PM [email protected]
> > > <[email protected]> wrote:
> > > >
> > > > On Friday, October 15, 2021 7:23 PM [email protected] wrote:
> > > > > Attach a patch to fix it.
> > > > Attach a new version patch which refactor the fix code in a cleaner way.
> > > >
> > >
> > > I have not debugged it yet to find out why, but with the patch
> > > applied, the original double-publish problem that I reported
> > > (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> > > occurs.
> > >
> >
> > Yeah, I think this is a variant of the problem being fixed by
> > Hou-San's patch. I think one possible idea to investigate is that on
> > the subscriber-side, after fetching tables, we check the already
> > subscribed tables and if the child tables already exist then we ignore
> > the parent table and vice versa. We might want to consider the case
> > where a user has toggled the "publish_via_partition_root" parameter.
> >
> > It seems both these behaviours/problems exist since commit 17b9e7f9
> > (Support adding partitioned tables to publication). Adding Amit L and
> > Peter E (people involved in this work) to know their opinion?
>
> I can imagine that the behavior seen here may look surprising, but not
> sure if I would call it a bug as such. I do remember thinking about
> this case and the current behavior is how I may have coded it to be.
>
> Looking at this command in Hou-san's email:
>
> create publication pub for table tbl1, tbl1_part1 with
> (publish_via_partition_root=on);
>
> It's adding both the root partitioned table and the leaf partition
> *explicitly*, and it's not clear to me if the latter's inclusion in
> the publication should be assumed because the former is found to have
> been added to the publication, that is, as far as the latter's
> visibility to the subscriber is concerned. It's not a stretch to
> imagine that a user may write the command this way to account for a
> subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
>
> I don't think we assume anything on the publisher side regarding the
> state/configuration of tables on the subscriber side, at least with
> publication commands where tables are added to a publication
> explicitly, so it is up to the user to make sure that the tables are
> not added duplicatively. One may however argue that the way we've
> decided to handle FOR ALL TABLES does assume something about
> partitions where it skips advertising them to subscribers when
> publish_via_partition_root flag is set to true, but that is exactly to
> avoid the duplication of data that goes to a subscriber.
>
I think the same confusion will then apply to the new feature (For All
Tables In Schema) being discussed [1] (that is a bit long thread so
shared the email where the latest patch version is posted). There
also, the partitioned table and partition can be in a different
schema. We either need to follow "For All Tables" or "For Table"
behavior. Then, there is also an argument that such behavior is not
documented, and by reading "publish_via_partition_root", it is not
clear why would the user expect the current behavior?
Also, what about Greg's case [2], where I think it is clear that the
subscriber also has partitions?
[1] - https://www.postgresql.org/message-id/OS0PR01MB5716B523961FE338EB9B3F9A94BC9%40OS0PR01MB5716.jpnprd0...
[2] - https://www.postgresql.org/message-id/CAJcOf-eQR_%3Dq0f4ZVHd342QdLvBd_995peSr4xCU05hrS3TeTg%40mail.g...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-19 02:47 [email protected] <[email protected]>
parent: Amit Langote <[email protected]>
1 sibling, 2 replies; 99+ messages in thread
From: [email protected] @ 2021-10-19 02:47 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>
On Monday, October 18, 2021 5:03 PM Amit Langote <[email protected]> wrote:
> I can imagine that the behavior seen here may look surprising, but not
> sure if I would call it a bug as such. I do remember thinking about
> this case and the current behavior is how I may have coded it to be.
>
> Looking at this command in Hou-san's email:
>
> create publication pub for table tbl1, tbl1_part1 with
> (publish_via_partition_root=on);
>
> It's adding both the root partitioned table and the leaf partition
> *explicitly*, and it's not clear to me if the latter's inclusion in
> the publication should be assumed because the former is found to have
> been added to the publication, that is, as far as the latter's
> visibility to the subscriber is concerned. It's not a stretch to
> imagine that a user may write the command this way to account for a
> subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
>
> I don't think we assume anything on the publisher side regarding the
> state/configuration of tables on the subscriber side, at least with
> publication commands where tables are added to a publication
> explicitly, so it is up to the user to make sure that the tables are
> not added duplicatively. One may however argue that the way we've
> decided to handle FOR ALL TABLES does assume something about
> partitions where it skips advertising them to subscribers when
> publish_via_partition_root flag is set to true, but that is exactly to
> avoid the duplication of data that goes to a subscriber.
Hi,
Thanks for the explanation.
I think one reason that I consider this behavior a bug is that: If we add
both the root partitioned table and the leaf partition explicitly to the
publication (and set publish_via_partition_root = on), the behavior of the
apply worker is inconsistent with the behavior of table sync worker.
In this case, all changes in the leaf the partition will be applied using the
identity and schema of the partitioned(root) table. But for the table sync, it
will execute table sync for both the leaf and the root table which cause
duplication of data.
Wouldn't it be better to make the behavior consistent here ?
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-19 03:15 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-10-19 03:15 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Mon, Oct 18, 2021 at 2:58 PM Amit Kapila <[email protected]> wrote:
>
> On Mon, Oct 18, 2021 at 2:32 PM Amit Langote <[email protected]> wrote:
> >
> > Looking at this command in Hou-san's email:
> >
> > create publication pub for table tbl1, tbl1_part1 with
> > (publish_via_partition_root=on);
> >
> > It's adding both the root partitioned table and the leaf partition
> > *explicitly*, and it's not clear to me if the latter's inclusion in
> > the publication should be assumed because the former is found to have
> > been added to the publication, that is, as far as the latter's
> > visibility to the subscriber is concerned. It's not a stretch to
> > imagine that a user may write the command this way to account for a
> > subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
> >
> > I don't think we assume anything on the publisher side regarding the
> > state/configuration of tables on the subscriber side, at least with
> > publication commands where tables are added to a publication
> > explicitly, so it is up to the user to make sure that the tables are
> > not added duplicatively. One may however argue that the way we've
> > decided to handle FOR ALL TABLES does assume something about
> > partitions where it skips advertising them to subscribers when
> > publish_via_partition_root flag is set to true, but that is exactly to
> > avoid the duplication of data that goes to a subscriber.
> >
>
> I think the same confusion will then apply to the new feature (For All
> Tables In Schema) being discussed [1] (that is a bit long thread so
> shared the email where the latest patch version is posted). There
> also, the partitioned table and partition can be in a different
> schema.
>
Sorry, I wanted to say that table partition and partitioned table can
be in the same schema. Now, if the user publishes all tables in a
schema, if we want to follow the "For All Tables" behavior then we
should skip the leaf table and publish only the parent table, OTOH, if
want to follow "For Table" behavior, we need to publish both
partitioned table and partition table. I feel it is better to be
consistent here in all three cases ("For Table", "For All Tables", and
"For All Tables In Schema") as it will be easier to explain and
document it.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-19 04:29 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: Amit Kapila @ 2021-10-19 04:29 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Tue, Oct 19, 2021 at 8:45 AM Amit Kapila <[email protected]> wrote:
>
> On Mon, Oct 18, 2021 at 2:58 PM Amit Kapila <[email protected]> wrote:
> >
> > On Mon, Oct 18, 2021 at 2:32 PM Amit Langote <[email protected]> wrote:
> > >
> > > Looking at this command in Hou-san's email:
> > >
> > > create publication pub for table tbl1, tbl1_part1 with
> > > (publish_via_partition_root=on);
> > >
> > > It's adding both the root partitioned table and the leaf partition
> > > *explicitly*, and it's not clear to me if the latter's inclusion in
> > > the publication should be assumed because the former is found to have
> > > been added to the publication, that is, as far as the latter's
> > > visibility to the subscriber is concerned. It's not a stretch to
> > > imagine that a user may write the command this way to account for a
> > > subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
> > >
> > > I don't think we assume anything on the publisher side regarding the
> > > state/configuration of tables on the subscriber side, at least with
> > > publication commands where tables are added to a publication
> > > explicitly, so it is up to the user to make sure that the tables are
> > > not added duplicatively. One may however argue that the way we've
> > > decided to handle FOR ALL TABLES does assume something about
> > > partitions where it skips advertising them to subscribers when
> > > publish_via_partition_root flag is set to true, but that is exactly to
> > > avoid the duplication of data that goes to a subscriber.
> > >
> >
> > I think the same confusion will then apply to the new feature (For All
> > Tables In Schema) being discussed [1] (that is a bit long thread so
> > shared the email where the latest patch version is posted). There
> > also, the partitioned table and partition can be in a different
> > schema.
> >
>
> Sorry, I wanted to say that table partition and partitioned table can
> be in the same schema. Now, if the user publishes all tables in a
> schema, if we want to follow the "For All Tables" behavior then we
> should skip the leaf table and publish only the parent table, OTOH, if
> want to follow "For Table" behavior, we need to publish both
> partitioned table and partition table. I feel it is better to be
> consistent here in all three cases ("For Table", "For All Tables", and
> "For All Tables In Schema") as it will be easier to explain and
> document it.
>
Thinking some more about it, I think we also have a problem when the
partitioned and partition tables are in different schemas especially
when the user created a publication having a combination of "For
Table" and "For All Tables In Schema", see below:
create schema sch1;
create schema sch2;
create table sch1.tbl1 (a int) partition by range ( a );
create table sch2.tbl1_part1 partition of sch1.tbl1 for values from
(1) to (101);
create table sch1.tbl1_part2 partition of sch1.tbl1 for values from
(101) to (200);
create publication mypub for table sch2.tbl1_part1, all tables in
schema sch1 WITH (publish_via_partition_root = true);
Now, here if we follow the rules of "For Table", then we should get
both partitioned and partition tables which will be different from the
case when both are in the same schema considering we follow "For All
Tables" behavior in "For All Tables In Schema" case.
The point is that as we extend the current feature, I think the
complications will increase if we don't have a consistent behavior for
all cases and it will also be difficult to explain it to users.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-19 05:37 Dilip Kumar <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Dilip Kumar @ 2021-10-19 05:37 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Langote <[email protected]>; Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>
On Tue, Oct 19, 2021 at 8:17 AM [email protected]
<[email protected]> wrote:
> Thanks for the explanation.
>
> I think one reason that I consider this behavior a bug is that: If we add
> both the root partitioned table and the leaf partition explicitly to the
> publication (and set publish_via_partition_root = on), the behavior of the
> apply worker is inconsistent with the behavior of table sync worker.
>
> In this case, all changes in the leaf the partition will be applied using the
> identity and schema of the partitioned(root) table. But for the table sync, it
> will execute table sync for both the leaf and the root table which cause
> duplication of data.
>
> Wouldn't it be better to make the behavior consistent here ?
I agree with the point, whether we are doing the initial sync or we
are doing transaction streaming the behavior should be the same. I
think the right behavior should be that even if user has given both
parent table and the child table in the published table list, it
should sync it only once, because consider the case where we add a
same table twice e.g (CREATE PUBLICATION mypub FOR TABLE t1,t1;) but
in that case also we consider this table only once and there will be
no duplicate data.
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 07:14 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-20 07:14 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Mon, Oct 18, 2021 at 5:00 PM Amit Kapila <[email protected]> wrote:
>
> > I have not debugged it yet to find out why, but with the patch
> > applied, the original double-publish problem that I reported
> > (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> > occurs.
> >
>
> Yeah, I think this is a variant of the problem being fixed by
> Hou-San's patch. I think one possible idea to investigate is that on
> the subscriber-side, after fetching tables, we check the already
> subscribed tables and if the child tables already exist then we ignore
> the parent table and vice versa. We might want to consider the case
> where a user has toggled the "publish_via_partition_root" parameter.
>
> It seems both these behaviours/problems exist since commit 17b9e7f9
> (Support adding partitioned tables to publication). Adding Amit L and
> Peter E (people involved in this work) to know their opinion?
>
Actually, at least with the scenario I gave steps for, after looking
at it again and debugging, I think that the behavior is understandable
and not a bug.
The reason is that the INSERTed data is first published though the
partitions, since initially there is no partitioned table in the
publication (so publish_via_partition_root=true doesn't have any
effect). But then adding the partitioned table to the publication and
refreshing the publication in the subscriber, the data is then
published "using the identity and schema of the partitioned table" due
to publish_via_partition_root=true. Note that the corresponding table
in the subscriber may well be a non-partitioned table (or the
partitions arranged differently) so the data does need to be
replicated again.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 08:02 Dilip Kumar <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: Dilip Kumar @ 2021-10-20 08:02 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: Amit Kapila <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 12:44 PM Greg Nancarrow <[email protected]> wrote:
>
> On Mon, Oct 18, 2021 at 5:00 PM Amit Kapila <[email protected]> wrote:
> >
> > > I have not debugged it yet to find out why, but with the patch
> > > applied, the original double-publish problem that I reported
> > > (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> > > occurs.
> > >
> >
> > Yeah, I think this is a variant of the problem being fixed by
> > Hou-San's patch. I think one possible idea to investigate is that on
> > the subscriber-side, after fetching tables, we check the already
> > subscribed tables and if the child tables already exist then we ignore
> > the parent table and vice versa. We might want to consider the case
> > where a user has toggled the "publish_via_partition_root" parameter.
> >
> > It seems both these behaviours/problems exist since commit 17b9e7f9
> > (Support adding partitioned tables to publication). Adding Amit L and
> > Peter E (people involved in this work) to know their opinion?
> >
>
> Actually, at least with the scenario I gave steps for, after looking
> at it again and debugging, I think that the behavior is understandable
> and not a bug.
> The reason is that the INSERTed data is first published though the
> partitions, since initially there is no partitioned table in the
> publication (so publish_via_partition_root=true doesn't have any
> effect). But then adding the partitioned table to the publication and
> refreshing the publication in the subscriber, the data is then
> published "using the identity and schema of the partitioned table" due
> to publish_via_partition_root=true. Note that the corresponding table
> in the subscriber may well be a non-partitioned table (or the
> partitions arranged differently) so the data does need to be
> replicated again.
I don't think this behavior is consistent, I mean for the initial sync
we will replicate the duplicate data, whereas for later streaming we
will only replicate it once. From the user POW, this behavior doesn't
look correct.
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 08:59 Amit Kapila <[email protected]>
parent: Dilip Kumar <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-10-20 08:59 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 1:32 PM Dilip Kumar <[email protected]> wrote:
>
> On Wed, Oct 20, 2021 at 12:44 PM Greg Nancarrow <[email protected]> wrote:
> >
> > On Mon, Oct 18, 2021 at 5:00 PM Amit Kapila <[email protected]> wrote:
> > >
> > > > I have not debugged it yet to find out why, but with the patch
> > > > applied, the original double-publish problem that I reported
> > > > (converted to just use TABLE rather than ALL TABLES IN SCHEMA) still
> > > > occurs.
> > > >
> > >
> > > Yeah, I think this is a variant of the problem being fixed by
> > > Hou-San's patch. I think one possible idea to investigate is that on
> > > the subscriber-side, after fetching tables, we check the already
> > > subscribed tables and if the child tables already exist then we ignore
> > > the parent table and vice versa. We might want to consider the case
> > > where a user has toggled the "publish_via_partition_root" parameter.
> > >
> > > It seems both these behaviours/problems exist since commit 17b9e7f9
> > > (Support adding partitioned tables to publication). Adding Amit L and
> > > Peter E (people involved in this work) to know their opinion?
> > >
> >
> > Actually, at least with the scenario I gave steps for, after looking
> > at it again and debugging, I think that the behavior is understandable
> > and not a bug.
> > The reason is that the INSERTed data is first published though the
> > partitions, since initially there is no partitioned table in the
> > publication (so publish_via_partition_root=true doesn't have any
> > effect). But then adding the partitioned table to the publication and
> > refreshing the publication in the subscriber, the data is then
> > published "using the identity and schema of the partitioned table" due
> > to publish_via_partition_root=true. Note that the corresponding table
> > in the subscriber may well be a non-partitioned table (or the
> > partitions arranged differently) so the data does need to be
> > replicated again.
>
Even if the partitions are arranged differently why would the user
expect the same data to be replicated twice?
> I don't think this behavior is consistent, I mean for the initial sync
> we will replicate the duplicate data, whereas for later streaming we
> will only replicate it once. From the user POW, this behavior doesn't
> look correct.
>
+1.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 09:00 Greg Nancarrow <[email protected]>
parent: Dilip Kumar <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-20 09:00 UTC (permalink / raw)
To: Dilip Kumar <[email protected]>; +Cc: Amit Kapila <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 7:02 PM Dilip Kumar <[email protected]> wrote:
>
> > Actually, at least with the scenario I gave steps for, after looking
> > at it again and debugging, I think that the behavior is understandable
> > and not a bug.
> > The reason is that the INSERTed data is first published though the
> > partitions, since initially there is no partitioned table in the
> > publication (so publish_via_partition_root=true doesn't have any
> > effect). But then adding the partitioned table to the publication and
> > refreshing the publication in the subscriber, the data is then
> > published "using the identity and schema of the partitioned table" due
> > to publish_via_partition_root=true. Note that the corresponding table
> > in the subscriber may well be a non-partitioned table (or the
> > partitions arranged differently) so the data does need to be
> > replicated again.
>
> I don't think this behavior is consistent, I mean for the initial sync
> we will replicate the duplicate data, whereas for later streaming we
> will only replicate it once. From the user POW, this behavior doesn't
> look correct.
>
The scenario I gave steps for didn't have any table data when the
subscription was made, so the initial sync did not replicate any data.
I was referring to the double-publish that occurs when
publish_via_partition_root=true and then the partitioned table is
added to the publication and the subscriber does ALTER SUBSCRIPTION
... REFRESH PUBLICATION.
If I modify my example to include both the partitioned table and
(explicitly) its child partitions in the publication, and insert some
data on the publisher side prior to the subscription, then I am seeing
duplicate data on the initial sync on the subscriber side, and I would
agree that this doesn't seem correct.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 09:33 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-20 09:33 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 7:59 PM Amit Kapila <[email protected]> wrote:
>
> > > Actually, at least with the scenario I gave steps for, after looking
> > > at it again and debugging, I think that the behavior is understandable
> > > and not a bug.
> > > The reason is that the INSERTed data is first published though the
> > > partitions, since initially there is no partitioned table in the
> > > publication (so publish_via_partition_root=true doesn't have any
> > > effect). But then adding the partitioned table to the publication and
> > > refreshing the publication in the subscriber, the data is then
> > > published "using the identity and schema of the partitioned table" due
> > > to publish_via_partition_root=true. Note that the corresponding table
> > > in the subscriber may well be a non-partitioned table (or the
> > > partitions arranged differently) so the data does need to be
> > > replicated again.
> >
>
> Even if the partitions are arranged differently why would the user
> expect the same data to be replicated twice?
>
It's the same data, but published in different ways because of changes
the user made to the publication.
I am not talking in general, I am specifically referring to the
scenario I gave steps for.
In the example scenario I gave, initially when the subscription was
made, the publication just explicitly included the partitions, but
publish_via_partition_root was true. So in this case it publishes
through the individual partitions (as no partitioned table is present
in the publication). Then on the publisher side, the partitioned table
was then added to the publication and then ALTER SUBSCRIPTION ...
REFRESH PUBLICATION done on the subscriber side. Now that the
partitioned table is present in the publication and
publish_via_partition_root is true, it is "published using the
identity and schema of the partitioned table rather than that of the
individual partitions that are actually changed". So the data is
replicated again.
This scenario didn't use initial table data, so initial table sync
didn't come into play (although as I previously posted, I can see a
double-publish issue on initial sync if data is put in the table prior
to subscription and partitions have been explicitly added to the
publication).
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 10:19 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-10-20 10:19 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 3:03 PM Greg Nancarrow <[email protected]> wrote:
>
> On Wed, Oct 20, 2021 at 7:59 PM Amit Kapila <[email protected]> wrote:
> >
> > > > Actually, at least with the scenario I gave steps for, after looking
> > > > at it again and debugging, I think that the behavior is understandable
> > > > and not a bug.
> > > > The reason is that the INSERTed data is first published though the
> > > > partitions, since initially there is no partitioned table in the
> > > > publication (so publish_via_partition_root=true doesn't have any
> > > > effect). But then adding the partitioned table to the publication and
> > > > refreshing the publication in the subscriber, the data is then
> > > > published "using the identity and schema of the partitioned table" due
> > > > to publish_via_partition_root=true. Note that the corresponding table
> > > > in the subscriber may well be a non-partitioned table (or the
> > > > partitions arranged differently) so the data does need to be
> > > > replicated again.
> > >
> >
> > Even if the partitions are arranged differently why would the user
> > expect the same data to be replicated twice?
> >
>
> It's the same data, but published in different ways because of changes
> the user made to the publication.
> I am not talking in general, I am specifically referring to the
> scenario I gave steps for.
> In the example scenario I gave, initially when the subscription was
> made, the publication just explicitly included the partitions, but
> publish_via_partition_root was true. So in this case it publishes
> through the individual partitions (as no partitioned table is present
> in the publication). Then on the publisher side, the partitioned table
> was then added to the publication and then ALTER SUBSCRIPTION ...
> REFRESH PUBLICATION done on the subscriber side. Now that the
> partitioned table is present in the publication and
> publish_via_partition_root is true, it is "published using the
> identity and schema of the partitioned table rather than that of the
> individual partitions that are actually changed". So the data is
> replicated again.
>
I don't see why data need to be replicated again even in that case.
Can you see any such duplicate data replicated for non-partitioned
tables?
> This scenario didn't use initial table data, so initial table sync
> didn't come into play
>
It will be equivalent to initial sync because the tablesync worker
would copy the entire data again in this case unless during refresh we
pass copy_data as false.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-20 13:40 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-20 13:40 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 9:19 PM Amit Kapila <[email protected]> wrote:
>
> I don't see why data need to be replicated again even in that case.
> Can you see any such duplicate data replicated for non-partitioned
> tables?
>
If my example is slightly modified to use the same-named tables on the
subscriber side, but without partitioning, i.e.:
PUB:
CREATE SCHEMA sch;
CREATE SCHEMA sch1;
CREATE TABLE sch.sale (sale_date date not null, country_code text,
product_sku text, units integer) PARTITION BY RANGE (sale_date);
CREATE TABLE sch1.sale_201901 PARTITION OF sch.sale FOR VALUES FROM
('2019-01-01') TO ('2019-02-01');
CREATE TABLE sch1.sale_201902 PARTITION OF sch.sale FOR VALUES FROM
('2019-02-01') TO ('2019-03-01');
SUB:
CREATE SCHEMA sch;
CREATE SCHEMA sch1;
CREATE TABLE sch.sale (sale_date date not null, country_code text,
product_sku text, units integer);
CREATE TABLE sch1.sale_201901 (sale_date date not null, country_code
text, product_sku text, units integer);
CREATE TABLE sch1.sale_201902 (sale_date date not null, country_code
text, product_sku text, units integer);
then the INSERTed data on the publisher side gets replicated to the
subscriber's "sch1.sale_201901" and "sch1.sale_201902" tables (only),
depending on the date values.
Now if the partitioned table is then added to the publication and
ALTER SUBSCRIPTION ... REFRESH PUBLICATION done by the subscriber,
then the current functionality is that the existing sch.sale data is
replicated (only) to the subscriber's "sch.sale" table (even though
data had been replicated previously to the "sch1.sale_201901" and
"sch1.sale_201902" tables, only).
So, just to be clear, you think that this current functionality isn't
correct (i.e. no data should be replicated on the REFRESH in this
case)?
I think it's debatable because here copy_data=true and sch.sale was
not a previously-subscribed table (so pre-existing data in that table
should be copied, in accordance with the current documentation).
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-21 03:56 Peter Smith <[email protected]>
parent: Greg Nancarrow <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Peter Smith @ 2021-10-21 03:56 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: Amit Kapila <[email protected]>; Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Hi,
I just wanted to bring to your attention an earlier thread [1] in
which I had already encountered/reported behaviour that is either
exactly the same or is closely related to what is being discussed in
this current thread. If it is different please take that into account
also.
------
[1] https://www.postgresql.org/message-id/CAHut%2BPvJMRB-ZyC80we2kiUFv4cVjmA6jxXpEMhm1rmz%3D1ryeA%40mail...
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-21 05:57 Greg Nancarrow <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: Greg Nancarrow @ 2021-10-21 05:57 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Thu, Oct 21, 2021 at 2:56 PM Peter Smith <[email protected]> wrote:
>
> I just wanted to bring to your attention an earlier thread [1] in
> which I had already encountered/reported behaviour that is either
> exactly the same or is closely related to what is being discussed in
> this current thread. If it is different please take that into account
> also.
>
> ------
> [1] https://www.postgresql.org/message-id/CAHut%2BPvJMRB-ZyC80we2kiUFv4cVjmA6jxXpEMhm1rmz%3D1ryeA%40mail...
>
Thanks, I was able to reproduce that behavior, which is similar (but
in that case the publish_via_partition_root flag is toggled with the
partitioned table present in the publication, whereas in the case
being discussed the presence of the partitioned table in the
publication is toggled with publish_via_partition_root always true).
What seems to happen internally when a partitioned table is published
is that when publish_via_partition_root=true the subscriber to that
publication is effectively subscribed to the parent partitioned table
but not the child partitions. If publish_via_partition_root is changed
to false and the subscriber refreshes, the partitioned table is
unsubscribed and it subscribes to the partitions. This explains why
data gets "re-copied" when this happens, because then it is
subscribing to a "new" table and copy_data=true by default.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-21 09:45 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: Amit Kapila @ 2021-10-21 09:45 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: Dilip Kumar <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
On Wed, Oct 20, 2021 at 7:11 PM Greg Nancarrow <[email protected]> wrote:
>
> On Wed, Oct 20, 2021 at 9:19 PM Amit Kapila <[email protected]> wrote:
> >
> > I don't see why data need to be replicated again even in that case.
> > Can you see any such duplicate data replicated for non-partitioned
> > tables?
> >
>
> If my example is slightly modified to use the same-named tables on the
> subscriber side, but without partitioning, i.e.:
>
> PUB:
>
> CREATE SCHEMA sch;
> CREATE SCHEMA sch1;
> CREATE TABLE sch.sale (sale_date date not null, country_code text,
> product_sku text, units integer) PARTITION BY RANGE (sale_date);
> CREATE TABLE sch1.sale_201901 PARTITION OF sch.sale FOR VALUES FROM
> ('2019-01-01') TO ('2019-02-01');
> CREATE TABLE sch1.sale_201902 PARTITION OF sch.sale FOR VALUES FROM
> ('2019-02-01') TO ('2019-03-01');
>
>
> SUB:
>
> CREATE SCHEMA sch;
> CREATE SCHEMA sch1;
> CREATE TABLE sch.sale (sale_date date not null, country_code text,
> product_sku text, units integer);
> CREATE TABLE sch1.sale_201901 (sale_date date not null, country_code
> text, product_sku text, units integer);
> CREATE TABLE sch1.sale_201902 (sale_date date not null, country_code
> text, product_sku text, units integer);
>
> then the INSERTed data on the publisher side gets replicated to the
> subscriber's "sch1.sale_201901" and "sch1.sale_201902" tables (only),
> depending on the date values.
> Now if the partitioned table is then added to the publication and
> ALTER SUBSCRIPTION ... REFRESH PUBLICATION done by the subscriber,
> then the current functionality is that the existing sch.sale data is
> replicated (only) to the subscriber's "sch.sale" table (even though
> data had been replicated previously to the "sch1.sale_201901" and
> "sch1.sale_201902" tables, only).
> So, just to be clear, you think that this current functionality isn't
> correct (i.e. no data should be replicated on the REFRESH in this
> case)?
>
Right, I don't think it is correct because it will behave differently
when the tables on the subscriber are partitioned. Also, the idea I
speculated in one of my above emails should be able to deal with this
case.
> I think it's debatable because here copy_data=true and sch.sale was
> not a previously-subscribed table (so pre-existing data in that table
> should be copied, in accordance with the current documentation).
>
What about the partition (child) table? In this case, the same data
will be present in two tables sch.sale and sch1.sale_201901 after you
have refreshed the publication, and then any future insertions will
only be inserted into parent table sch.sale in this case which doesn't
sound consistent. The bigger problem is that it will lead to duplicate
data when tables are partitioned. I think if the user really wants to
do in a way you are describing, there is no need to keep sub-tables
(*_201901 and *__201902). I understand that it depends on the use case
but we should also behave sanely when tables/partitions are created in
the same way in both publisher and subscriber which I guess will most
likely be the case.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-22 02:01 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: [email protected] @ 2021-10-22 02:01 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Langote <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>
On Tuesday, October 19, 2021 10:47 AM [email protected] <[email protected]> wrote:
>
> On Monday, October 18, 2021 5:03 PM Amit Langote
> <[email protected]> wrote:
> > I can imagine that the behavior seen here may look surprising, but not
> > sure if I would call it a bug as such. I do remember thinking about
> > this case and the current behavior is how I may have coded it to be.
> >
> > Looking at this command in Hou-san's email:
> >
> > create publication pub for table tbl1, tbl1_part1 with
> > (publish_via_partition_root=on);
> >
> > It's adding both the root partitioned table and the leaf partition
> > *explicitly*, and it's not clear to me if the latter's inclusion in
> > the publication should be assumed because the former is found to have
> > been added to the publication, that is, as far as the latter's
> > visibility to the subscriber is concerned. It's not a stretch to
> > imagine that a user may write the command this way to account for a
> > subscriber node on which tbl1 and tbl1_part1 are unrelated tables.
> >
> > I don't think we assume anything on the publisher side regarding the
> > state/configuration of tables on the subscriber side, at least with
> > publication commands where tables are added to a publication
> > explicitly, so it is up to the user to make sure that the tables are
> > not added duplicatively. One may however argue that the way we've
> > decided to handle FOR ALL TABLES does assume something about
> > partitions where it skips advertising them to subscribers when
> > publish_via_partition_root flag is set to true, but that is exactly to
> > avoid the duplication of data that goes to a subscriber.
>
> Hi,
>
> Thanks for the explanation.
>
> I think one reason that I consider this behavior a bug is that: If we add
> both the root partitioned table and the leaf partition explicitly to the
> publication (and set publish_via_partition_root = on), the behavior of the
> apply worker is inconsistent with the behavior of table sync worker.
>
> In this case, all changes in the leaf the partition will be applied using the
> identity and schema of the partitioned(root) table. But for the table sync, it
> will execute table sync for both the leaf and the root table which cause
> duplication of data.
>
> Wouldn't it be better to make the behavior consistent here ?
>
I agree with this point.
About this case,
> > create publication pub for table tbl1, tbl1_part1 with
> > (publish_via_partition_root=on);
As a user, although partitioned table includes the partition, publishing partitioned
table and its partition is allowed. So, I think we should take this case into
consideration. Initial data is copied once via the parent table seems reasonable.
Regards
Shi yu
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-28 07:35 [email protected] <[email protected]>
parent: Dilip Kumar <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2021-10-28 07:35 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
Hi,
As there are basically two separate issues mentioned in the thread, I tried to
summarize the discussion so far which might be helpful to others.
* The first issue[1]:
If we include both the partitioned table and (explicitly) its child partitions
in the publication when set publish_via_partition_root=true, like:
---
CREATE PUBLICATION pub FOR TABLE parent_table, child_table with (publish_via_partition_root=on);
---
It could execute initial sync for both the partitioned(parent_table) table and
(explicitly) its child partitions(child_table) which cause duplication of
data in partition(child_table) in subscriber side.
The reasons I considered this behavior a bug are:
a) In this case, the behavior of initial sync is inconsistent with the behavior
of transaction streaming. All changes in the leaf the partition will be applied
using the identity and schema of the partitioned(root) table. But for the
initial sync, it will execute sync for both the partitioned(root) table and
(explicitly) its child partitions which cause duplication of data.
b) The behavior of FOR TABLE is inconsistent with the behavior of FOR ALL TABLE.
If user create a FOR ALL TABLE publication and set publish_via_partition_root=true,
then only the top most partitioned(root) table will execute initial sync.
IIRC, most people in this thread agreed that the current behavior is not
expected. So, maybe it's time to try to fix it.
Attach my fix patch here. The patch try to fix this issue by making the
pg_publication_tables view only show partitioned table when
publish_via_partition_root is true.
* The second issue[2]:
-----
CREATE TABLE sale (sale_date date not null,country_code text, product_sku text,
units integer) PARTITION BY RANGE (sale_date);
CREATE TABLE sale_201901 PARTITION OF sale FOR VALUES FROM ('2019-01-01') TO
('2019-02-01');
CREATE TABLE sale_201902 PARTITION OF sale FOR VALUES FROM ('2019-02-01') TO
('2019-03-01');
(1) PUB: CREATE PUBLICATION pub FOR TABLE sale_201901,
sale_201902 WITH (publish_via_partition_root=true);
(2) SUB: CREATE SUBSCRIPTION sub CONNECTION 'dbname=postgres host=localhost port=5432' PUBLICATION pub;
(3) PUB: INSERT INTO sale VALUES('2019-01-01', 'AU', 'cpu', 5), ('2019-01-02', 'AU', 'disk', 8);
(4) SUB: SELECT * FROM sale;
(5) PUB: ALTER PUBLICATION pub ADD TABLE sale;
(6) SUB: ALTER SUBSCRIPTION sub REFRESH PUBLICATION;
(7) SUB: SELECT * FROM sale;
-----
In step (7), we can see duplication of data.
The reason is that the INSERTed data is first published though the partitions,
since initially there is no partitioned table in the publication (so
publish_via_partition_root=true doesn't have any effect). But then adding the
partitioned table to the publication and refreshing the publication in the
subscriber, the data is then published "using the identity and schema of the
partitioned table" due to publish_via_partition_root=true.
(Copied from Greg's analysis).
Whether this behavior is correct is still under debate.
Overall, I think the second issue still needs further discussion while the
first issue seems clear that most people think it's unexpected. So, I think it
might be better to fix the first issue.
[1] https://www.postgresql.org/message-id/OS0PR01MB57167F45D481F78CDC5986F794B99%40OS0PR01MB5716.jpnprd0...
[2] https://www.postgresql.org/message-id/flat/CAJcOf-d8SWk3z3fJaLW9yuVux%3D2ESTsXOSdKzCq1O3AWBpgnMQ%40m...
Best regards,
Hou zj
Attachments:
[application/octet-stream] v4-0001-fix-data-double-published.patch (4.3K, ../../OS0PR01MB5716C756312959F293A822C794869@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v4-0001-fix-data-double-published.patch)
download | inline diff:
From 6fdcae7ffda13d81de73414b2c405f5bbd6274c5 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 28 Oct 2021 13:58:51 +0800
Subject: [PATCH] Fix double publish of child table's data.
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 18 ++++--------------
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
3 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index fed83b89a9..019ee16d36 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -136,7 +136,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -157,14 +157,8 @@ filter_partitions(List *relids, List *schemarelids)
/*
* Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
*/
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -873,7 +867,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
+ if (publication->pubviaroot)
{
/*
* If the publication publishes partition changes via their
@@ -882,12 +876,8 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* tables. Otherwise, the function could return both the child
* and parent tables which could cause data of the child table
* to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
*/
- tables = filter_partitions(tables, schemarelids);
+ tables = filter_partitions(tables);
}
}
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 0f4fe4db8f..55777223ce 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -798,6 +798,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 85a5302a74..04e462d371 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -445,6 +445,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-10-28 08:25 Amit Langote <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Langote @ 2021-10-28 08:25 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Oct 28, 2021 at 4:35 PM [email protected]
<[email protected]> wrote:
> As there are basically two separate issues mentioned in the thread, I tried to
> summarize the discussion so far which might be helpful to others.
>
> * The first issue[1]:
>
> If we include both the partitioned table and (explicitly) its child partitions
> in the publication when set publish_via_partition_root=true, like:
> ---
> CREATE PUBLICATION pub FOR TABLE parent_table, child_table with (publish_via_partition_root=on);
> ---
> It could execute initial sync for both the partitioned(parent_table) table and
> (explicitly) its child partitions(child_table) which cause duplication of
> data in partition(child_table) in subscriber side.
>
> The reasons I considered this behavior a bug are:
>
> a) In this case, the behavior of initial sync is inconsistent with the behavior
> of transaction streaming. All changes in the leaf the partition will be applied
> using the identity and schema of the partitioned(root) table. But for the
> initial sync, it will execute sync for both the partitioned(root) table and
> (explicitly) its child partitions which cause duplication of data.
>
> b) The behavior of FOR TABLE is inconsistent with the behavior of FOR ALL TABLE.
> If user create a FOR ALL TABLE publication and set publish_via_partition_root=true,
> then only the top most partitioned(root) table will execute initial sync.
>
> IIRC, most people in this thread agreed that the current behavior is not
> expected. So, maybe it's time to try to fix it.
>
> Attach my fix patch here. The patch try to fix this issue by making the
> pg_publication_tables view only show partitioned table when
> publish_via_partition_root is true.
>
>
> * The second issue[2]:
> -----
> CREATE TABLE sale (sale_date date not null,country_code text, product_sku text,
> units integer) PARTITION BY RANGE (sale_date);
> CREATE TABLE sale_201901 PARTITION OF sale FOR VALUES FROM ('2019-01-01') TO
> ('2019-02-01');
> CREATE TABLE sale_201902 PARTITION OF sale FOR VALUES FROM ('2019-02-01') TO
> ('2019-03-01');
>
> (1) PUB: CREATE PUBLICATION pub FOR TABLE sale_201901,
> sale_201902 WITH (publish_via_partition_root=true);
> (2) SUB: CREATE SUBSCRIPTION sub CONNECTION 'dbname=postgres host=localhost port=5432' PUBLICATION pub;
> (3) PUB: INSERT INTO sale VALUES('2019-01-01', 'AU', 'cpu', 5), ('2019-01-02', 'AU', 'disk', 8);
> (4) SUB: SELECT * FROM sale;
> (5) PUB: ALTER PUBLICATION pub ADD TABLE sale;
> (6) SUB: ALTER SUBSCRIPTION sub REFRESH PUBLICATION;
> (7) SUB: SELECT * FROM sale;
> -----
>
> In step (7), we can see duplication of data.
>
> The reason is that the INSERTed data is first published though the partitions,
> since initially there is no partitioned table in the publication (so
> publish_via_partition_root=true doesn't have any effect). But then adding the
> partitioned table to the publication and refreshing the publication in the
> subscriber, the data is then published "using the identity and schema of the
> partitioned table" due to publish_via_partition_root=true.
> (Copied from Greg's analysis).
>
> Whether this behavior is correct is still under debate.
>
>
> Overall, I think the second issue still needs further discussion while the
> first issue seems clear that most people think it's unexpected. So, I think it
> might be better to fix the first issue.
Thanks for the summary, Hou-san, and sorry about my late reply.
I had thought about this some last week and I am coming around to
recognizing the confusing user experience of the current behavior.
Though, I am not sure if removing partitions from the result of
pg_publication_tables view for pubviaroot publications is acceptable
as a fix, because that prevents replicating into a subscriber node
where tables that are partition root and a partition respectively on
the publisher are independent tables on the subscriber. ISTM that
what Amit K mentioned in his first reply may be closer to what we may
ultimately need to do, which is this:
"I think one possible idea to investigate is that on the
subscriber-side, after fetching tables, we check the already
subscribed tables and if the child tables already exist then we ignore
the parent table and vice versa."
I had also thought about a way to implement that a bit and part of
that is to make pg_publication_tables always expose leaf partitions,
that is, even if pubviaroot is true for a given publication. So, when
puviaroot is true we include the partition root actually mentioned in
the publication as we do currently (changes streamed after initial
sync will always use its schema so the subscriber better know about
it), and also leaf partitions which we currently don't. The latter
too so that a subscriber can determine which leaf partitions to sync
and which ones to not based on whether a given leaf partitions is
already known to the subscriber, which means their pg_subscription_rel
entries would need to be made which we don't currently. Leaf
partition state entries will not be referred to after all of the known
ones are synced, because the subsequent changes will be streamed with
the root's schema. The initial sync worker would need to be taught to
skip processing any partitioned tables that it sees, because now we'd
be handling the initial sync part via individual leaf partitions.
Though the thing that makes this a bit tricky to implement is that the
pg_publication_tables view exposes way less information to be able to
determine how the tables listed are related to each other. So I think
we'd need to fix the view to return leaf partitions' owning root table
(if they're published through a pubviaroot publication) or rewrite the
query that fetch_table_list() uses to access the underlying catalogs
directly in the back-ported version.
That is roughly how I'd go about this. Thoughts?
--
Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-03 10:13 Amit Kapila <[email protected]>
parent: Amit Langote <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-11-03 10:13 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: [email protected] <[email protected]>; Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Oct 28, 2021 at 1:55 PM Amit Langote <[email protected]> wrote:
>
> On Thu, Oct 28, 2021 at 4:35 PM [email protected]
> <[email protected]> wrote:
> > As there are basically two separate issues mentioned in the thread, I tried to
> > summarize the discussion so far which might be helpful to others.
> >
> > * The first issue[1]:
> >
> > If we include both the partitioned table and (explicitly) its child partitions
> > in the publication when set publish_via_partition_root=true, like:
> > ---
> > CREATE PUBLICATION pub FOR TABLE parent_table, child_table with (publish_via_partition_root=on);
> > ---
> > It could execute initial sync for both the partitioned(parent_table) table and
> > (explicitly) its child partitions(child_table) which cause duplication of
> > data in partition(child_table) in subscriber side.
> >
> > The reasons I considered this behavior a bug are:
> >
> > a) In this case, the behavior of initial sync is inconsistent with the behavior
> > of transaction streaming. All changes in the leaf the partition will be applied
> > using the identity and schema of the partitioned(root) table. But for the
> > initial sync, it will execute sync for both the partitioned(root) table and
> > (explicitly) its child partitions which cause duplication of data.
> >
> > b) The behavior of FOR TABLE is inconsistent with the behavior of FOR ALL TABLE.
> > If user create a FOR ALL TABLE publication and set publish_via_partition_root=true,
> > then only the top most partitioned(root) table will execute initial sync.
> >
> > IIRC, most people in this thread agreed that the current behavior is not
> > expected. So, maybe it's time to try to fix it.
> >
> > Attach my fix patch here. The patch try to fix this issue by making the
> > pg_publication_tables view only show partitioned table when
> > publish_via_partition_root is true.
> >
> >
> > * The second issue[2]:
> > -----
> > CREATE TABLE sale (sale_date date not null,country_code text, product_sku text,
> > units integer) PARTITION BY RANGE (sale_date);
> > CREATE TABLE sale_201901 PARTITION OF sale FOR VALUES FROM ('2019-01-01') TO
> > ('2019-02-01');
> > CREATE TABLE sale_201902 PARTITION OF sale FOR VALUES FROM ('2019-02-01') TO
> > ('2019-03-01');
> >
> > (1) PUB: CREATE PUBLICATION pub FOR TABLE sale_201901,
> > sale_201902 WITH (publish_via_partition_root=true);
> > (2) SUB: CREATE SUBSCRIPTION sub CONNECTION 'dbname=postgres host=localhost port=5432' PUBLICATION pub;
> > (3) PUB: INSERT INTO sale VALUES('2019-01-01', 'AU', 'cpu', 5), ('2019-01-02', 'AU', 'disk', 8);
> > (4) SUB: SELECT * FROM sale;
> > (5) PUB: ALTER PUBLICATION pub ADD TABLE sale;
> > (6) SUB: ALTER SUBSCRIPTION sub REFRESH PUBLICATION;
> > (7) SUB: SELECT * FROM sale;
> > -----
> >
> > In step (7), we can see duplication of data.
> >
> > The reason is that the INSERTed data is first published though the partitions,
> > since initially there is no partitioned table in the publication (so
> > publish_via_partition_root=true doesn't have any effect). But then adding the
> > partitioned table to the publication and refreshing the publication in the
> > subscriber, the data is then published "using the identity and schema of the
> > partitioned table" due to publish_via_partition_root=true.
> > (Copied from Greg's analysis).
> >
> > Whether this behavior is correct is still under debate.
> >
> >
> > Overall, I think the second issue still needs further discussion while the
> > first issue seems clear that most people think it's unexpected. So, I think it
> > might be better to fix the first issue.
>
> Thanks for the summary, Hou-san, and sorry about my late reply.
>
> I had thought about this some last week and I am coming around to
> recognizing the confusing user experience of the current behavior.
> Though, I am not sure if removing partitions from the result of
> pg_publication_tables view for pubviaroot publications is acceptable
> as a fix, because that prevents replicating into a subscriber node
> where tables that are partition root and a partition respectively on
> the publisher are independent tables on the subscriber.
>
But we already do that way when the publication is "For All Tables".
Anyway, for the purpose of initial sync, it will just replicate the
same data in two different tables if the corresponding tables on the
subscriber-side are non-partitioned which I am not sure is what the
user will be expecting.
> ISTM that
> what Amit K mentioned in his first reply may be closer to what we may
> ultimately need to do, which is this:
>
> "I think one possible idea to investigate is that on the
> subscriber-side, after fetching tables, we check the already
> subscribed tables and if the child tables already exist then we ignore
> the parent table and vice versa."
>
> I had also thought about a way to implement that a bit and part of
> that is to make pg_publication_tables always expose leaf partitions,
> that is, even if pubviaroot is true for a given publication. So, when
> puviaroot is true we include the partition root actually mentioned in
> the publication as we do currently (changes streamed after initial
> sync will always use its schema so the subscriber better know about
> it), and also leaf partitions which we currently don't. The latter
> too so that a subscriber can determine which leaf partitions to sync
> and which ones to not based on whether a given leaf partitions is
> already known to the subscriber, which means their pg_subscription_rel
> entries would need to be made which we don't currently. Leaf
> partition state entries will not be referred to after all of the known
> ones are synced, because the subsequent changes will be streamed with
> the root's schema. The initial sync worker would need to be taught to
> skip processing any partitioned tables that it sees, because now we'd
> be handling the initial sync part via individual leaf partitions.
>
Yeah, we can do something like this as well but my guess is that this
will be a bit complicated. OTOH, if we adopt Hou-san's patch for the
first problem as described in his previous email then I think the
solution for the second problem could be simpler than what you
described above.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-04 04:13 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-11-04 04:13 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: [email protected] <[email protected]>; Greg Nancarrow <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Wed, Nov 3, 2021 at 3:43 PM Amit Kapila <[email protected]> wrote:
>
> On Thu, Oct 28, 2021 at 1:55 PM Amit Langote <[email protected]> wrote:
> >
> >
> > Thanks for the summary, Hou-san, and sorry about my late reply.
> >
> > I had thought about this some last week and I am coming around to
> > recognizing the confusing user experience of the current behavior.
> > Though, I am not sure if removing partitions from the result of
> > pg_publication_tables view for pubviaroot publications is acceptable
> > as a fix, because that prevents replicating into a subscriber node
> > where tables that are partition root and a partition respectively on
> > the publisher are independent tables on the subscriber.
> >
>
> But we already do that way when the publication is "For All Tables".
> Anyway, for the purpose of initial sync, it will just replicate the
> same data in two different tables if the corresponding tables on the
> subscriber-side are non-partitioned which I am not sure is what the
> user will be expecting.
>
On further thinking about this, I think we should define the behavior
of replication among partitioned (on the publisher) and
non-partitioned (on the subscriber) tables a bit more clearly.
- If the "publish_via_partition_root" is set for a publication then we
can always replicate to the table with the same name as the root table
in publisher.
- If the "publish_via_partition_root" is *not* set for a publication
then we can always replicate to the tables with the same name as the
non-root tables in publisher.
Thoughts?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-04 06:53 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-11-04 06:53 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; [email protected] <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 4, 2021 at 3:13 PM Amit Kapila <[email protected]> wrote:
>
> On further thinking about this, I think we should define the behavior
> of replication among partitioned (on the publisher) and
> non-partitioned (on the subscriber) tables a bit more clearly.
>
> - If the "publish_via_partition_root" is set for a publication then we
> can always replicate to the table with the same name as the root table
> in publisher.
> - If the "publish_via_partition_root" is *not* set for a publication
> then we can always replicate to the tables with the same name as the
> non-root tables in publisher.
>
> Thoughts?
>
I'd adjust that wording slightly, because "we can always replicate to
..." sounds a bit vague, and saying that an option is set or not set
could be misinterpreted, as the option could be "set" to false.
How about:
- If "publish_via_partition_root" is true for a publication, then data
is replicated to the table with the same name as the root (i.e.
partitioned) table in the publisher.
- If "publish_via_partition_root" is false (the default) for a
publication, then data is replicated to tables with the same name as
the non-root (i.e. partition) tables in the publisher.
?
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-04 08:10 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-11-04 08:10 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: Amit Langote <[email protected]>; [email protected] <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 4, 2021 at 12:23 PM Greg Nancarrow <[email protected]> wrote:
>
> On Thu, Nov 4, 2021 at 3:13 PM Amit Kapila <[email protected]> wrote:
> >
> > On further thinking about this, I think we should define the behavior
> > of replication among partitioned (on the publisher) and
> > non-partitioned (on the subscriber) tables a bit more clearly.
> >
> > - If the "publish_via_partition_root" is set for a publication then we
> > can always replicate to the table with the same name as the root table
> > in publisher.
> > - If the "publish_via_partition_root" is *not* set for a publication
> > then we can always replicate to the tables with the same name as the
> > non-root tables in publisher.
> >
> > Thoughts?
> >
>
> I'd adjust that wording slightly, because "we can always replicate to
> ..." sounds a bit vague, and saying that an option is set or not set
> could be misinterpreted, as the option could be "set" to false.
>
> How about:
>
> - If "publish_via_partition_root" is true for a publication, then data
> is replicated to the table with the same name as the root (i.e.
> partitioned) table in the publisher.
> - If "publish_via_partition_root" is false (the default) for a
> publication, then data is replicated to tables with the same name as
> the non-root (i.e. partition) tables in the publisher.
>
Sounds good to me. If we follow this then I think the patch by Hou-San
is good to solve the first problem as described in his last email [1]?
[1] - https://www.postgresql.org/message-id/OS0PR01MB5716C756312959F293A822C794869%40OS0PR01MB5716.jpnprd0...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-05 03:20 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-11-05 03:20 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; [email protected] <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 4, 2021 at 7:10 PM Amit Kapila <[email protected]> wrote:
>
> On Thu, Nov 4, 2021 at 12:23 PM Greg Nancarrow <[email protected]>
wrote:
> >
> > On Thu, Nov 4, 2021 at 3:13 PM Amit Kapila <[email protected]>
wrote:
> > >
> > > On further thinking about this, I think we should define the behavior
> > > of replication among partitioned (on the publisher) and
> > > non-partitioned (on the subscriber) tables a bit more clearly.
> > >
> > > - If the "publish_via_partition_root" is set for a publication then we
> > > can always replicate to the table with the same name as the root table
> > > in publisher.
> > > - If the "publish_via_partition_root" is *not* set for a publication
> > > then we can always replicate to the tables with the same name as the
> > > non-root tables in publisher.
> > >
> > > Thoughts?
> > >
> >
> > I'd adjust that wording slightly, because "we can always replicate to
> > ..." sounds a bit vague, and saying that an option is set or not set
> > could be misinterpreted, as the option could be "set" to false.
> >
> > How about:
> >
> > - If "publish_via_partition_root" is true for a publication, then data
> > is replicated to the table with the same name as the root (i.e.
> > partitioned) table in the publisher.
> > - If "publish_via_partition_root" is false (the default) for a
> > publication, then data is replicated to tables with the same name as
> > the non-root (i.e. partition) tables in the publisher.
> >
>
> Sounds good to me. If we follow this then I think the patch by Hou-San
> is good to solve the first problem as described in his last email [1]?
>
> [1] -
https://www.postgresql.org/message-id/OS0PR01MB5716C756312959F293A822C794869%40OS0PR01MB5716.jpnprd0...
>
Almost.
The patch does seem to solve that first problem (double publish on
tablesync).
I used the following test (taken from [2]), and variations of it:
--- Setup
create schema sch1;
create schema sch2;
create table sch1.tbl1 (a int) partition by range (a);
create table sch2.tbl1_part1 partition of sch1.tbl1 for values from (1) to
(10);
create table sch2.tbl1_part2 partition of sch1.tbl1 for values from
(10) to (20);
create schema sch3;
create table sch3.t1(c1 int);
--- Publication
create publication pub1 for all tables in schema sch3, table
sch1.tbl1, table sch2.tbl1_part1 with ( publish_via_partition_root=on);
insert into sch1.tbl1 values(1);
insert into sch1.tbl1 values(11);
insert into sch3.t1 values(1);
---- Subscription
CREATE SUBSCRIPTION sub CONNECTION 'dbname=postgres host=localhost
port=5432' PUBLICATION pub1;
[2] -
https://postgr.es/m/CALDaNm3vxjPMMSrVDNK0f8UWP+EQ5ry14xfEukmXsVg_UcwZNA@mail.gmail.com
However, there did still seem to be a problem, if
publish_via_partition_root is then set to false; it seems that can result
in duplicate partition entries in the pg_publication_tables view, see below
(this follows on from the test scenario given above):
postgres=# select * from pg_publication_tables;
pubname | schemaname | tablename
---------+------------+-----------
pub1 | sch1 | tbl1
pub1 | sch3 | t1
(2 rows)
postgres=# alter publication pub1 set (publish_via_partition_root=false);
ALTER PUBLICATION
postgres=# select * from pg_publication_tables;
pubname | schemaname | tablename
---------+------------+------------
pub1 | sch2 | tbl1_part1
pub1 | sch2 | tbl1_part2
pub1 | sch2 | tbl1_part1
pub1 | sch3 | t1
(4 rows)
So I think the patch would need to be updated to prevent that.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-11 06:52 [email protected] <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 3 replies; 99+ messages in thread
From: [email protected] @ 2021-11-11 06:52 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Friday, November 5, 2021 11:20 AM Greg Nancarrow <[email protected]> wrote:
>On Thu, Nov 4, 2021 at 7:10 PM Amit Kapila <mailto:[email protected]> wrote:
>>
>> On Thu, Nov 4, 2021 at 12:23 PM Greg Nancarrow <mailto:[email protected]> wrote:
>> >
>> > On Thu, Nov 4, 2021 at 3:13 PM Amit Kapila <mailto:[email protected]> wrote:
>> > >
>> > > On further thinking about this, I think we should define the behavior
>> > > of replication among partitioned (on the publisher) and
>> > > non-partitioned (on the subscriber) tables a bit more clearly.
>> > >
>> > > - If the "publish_via_partition_root" is set for a publication then we
>> > > can always replicate to the table with the same name as the root table
>> > > in publisher.
>> > > - If the "publish_via_partition_root" is *not* set for a publication
>> > > then we can always replicate to the tables with the same name as the
>> > > non-root tables in publisher.
>> > >
>> > > Thoughts?
>> > >
>> >
>> > I'd adjust that wording slightly, because "we can always replicate to
>> > ..." sounds a bit vague, and saying that an option is set or not set
>> > could be misinterpreted, as the option could be "set" to false.
>> >
>> > How about:
>> >
>> > - If "publish_via_partition_root" is true for a publication, then data
>> > is replicated to the table with the same name as the root (i.e.
>> > partitioned) table in the publisher.
>> > - If "publish_via_partition_root" is false (the default) for a
>> > publication, then data is replicated to tables with the same name as
>> > the non-root (i.e. partition) tables in the publisher.
>> >
>>
>> Sounds good to me. If we follow this then I think the patch by Hou-San
>> is good to solve the first problem as described in his last email [1]?
>>
>> [1] - https://www.postgresql.org/message-id/OS0PR01MB5716C756312959F293A822C794869%40OS0PR01MB5716.jpnprd0...
>>
>
>Almost.
>The patch does seem to solve that first problem (double publish on tablesync).
>I used the following test (taken from [2]), and variations of it:
>
>However, there did still seem to be a problem, if publish_via_partition_root is then set to false; it seems that can result in
>duplicate partition entries in the pg_publication_tables view, see below (this follows on from the test scenario given above):
>
>postgres=# select * from pg_publication_tables;
> pubname | schemaname | tablename
>---------+------------+-----------
> pub1 | sch1 | tbl1
> pub1 | sch3 | t1
>(2 rows)
>
>postgres=# alter publication pub1 set (publish_via_partition_root=false);
>ALTER PUBLICATION
>postgres=# select * from pg_publication_tables;
> pubname | schemaname | tablename
>---------+------------+------------
> pub1 | sch2 | tbl1_part1
> pub1 | sch2 | tbl1_part2
> pub1 | sch2 | tbl1_part1
> pub1 | sch3 | t1
>(4 rows)
>
>So I think the patch would need to be updated to prevent that.
Thanks for testing the patch.
The reason of the duplicate output is that:
The existing function GetPublicationRelations doesn't de-duplicate the output
oid list. So, when adding both child and parent table to the
publication(pubviaroot = false), the pg_publication_tables view will output
duplicate partition.
Attach the fix patch.
0001 fix data double publish(first issue in this thread)
0002 fix duplicate partition in view pg_publication_tables(reported by greg when testing the 0001 patch)
About the fix for second issue in this thread.
> "I think one possible idea to investigate is that on the
> subscriber-side, after fetching tables, we check the already
> subscribed tables and if the child tables already exist then we ignore
> the parent table and vice versa."
When looking into how to fix the second issue, I have a question:
After changing publish_via_partition_root from false to true, the
subcriber will fetch the partitioned table from publisher when refreshing.
In subsriber side, If all the child tables of the partitioned table already
subscribed, then we can just skip the table sync for the partitioned table. But
if only some of the child tables(not all child tables) were already subscribed,
should we skip the partitioned table's table sync ? I am not sure about the
appropriate behavior here.
What do you think ?
Best regards,
Hou zj
Attachments:
[application/octet-stream] v5-0002-fix-duplicate-table-in-pg_publication_tables.patch (2.8K, ../../OS0PR01MB5716B73A9B3C87466CD08A1D94949@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v5-0002-fix-duplicate-table-in-pg_publication_tables.patch)
download | inline diff:
From fccf6783936dc7293d847ede008185e236f27d0a Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 11 Nov 2021 13:43:51 +0800
Subject: [PATCH] fix duplicate table in pg_publication_tables
if publish_via_partition_root is false, and both child and parent
table were added to the publication. Then there could be duplicate child
table in the view pg_publication_tables. The reason is that the function
GetPublicationRelations doesn't de-duplicate the output Oid List.
Fix it by de-duplicating the relation oid in GetPublicationRelations.
---
src/backend/catalog/pg_publication.c | 5 ++++-
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index fed83b89a9..933456cfd0 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -472,10 +472,13 @@ GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt)
while (HeapTupleIsValid(tup = systable_getnext(scan)))
{
Form_pg_publication_rel pubrel;
+ List *relids = NIL;
pubrel = (Form_pg_publication_rel) GETSTRUCT(tup);
- result = GetPubPartitionOptionRelations(result, pub_partopt,
+ relids = GetPubPartitionOptionRelations(relids, pub_partopt,
pubrel->prrelid);
+
+ result = list_concat_unique_oid(result, relids);
}
systable_endscan(scan);
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 2ff21a7543..af3da0a855 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -816,6 +816,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+------------
+ pub | sch2 | tbl1_part1
+(1 row)
+
DROP PUBLICATION pub;
DROP TABLE sch2.tbl1_part1;
DROP TABLE sch1.tbl1;
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 85a5302a74..829407aea3 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -455,6 +455,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
DROP TABLE sch2.tbl1_part1;
DROP TABLE sch1.tbl1;
--
2.18.4
[application/octet-stream] v5-0001-fix-data-double-published.patch (4.3K, ../../OS0PR01MB5716B73A9B3C87466CD08A1D94949@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v5-0001-fix-data-double-published.patch)
download | inline diff:
From 6fdcae7ffda13d81de73414b2c405f5bbd6274c5 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 28 Oct 2021 13:58:51 +0800
Subject: [PATCH] Fix double publish of child table's data.
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 18 ++++--------------
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
3 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index fed83b89a9..019ee16d36 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -136,7 +136,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -157,14 +157,8 @@ filter_partitions(List *relids, List *schemarelids)
/*
* Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
*/
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -873,7 +867,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
+ if (publication->pubviaroot)
{
/*
* If the publication publishes partition changes via their
@@ -882,12 +876,8 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* tables. Otherwise, the function could return both the child
* and parent tables which could cause data of the child table
* to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
*/
- tables = filter_partitions(tables, schemarelids);
+ tables = filter_partitions(tables);
}
}
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 0f4fe4db8f..55777223ce 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -798,6 +798,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 85a5302a74..04e462d371 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -445,6 +445,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-11 08:14 Greg Nancarrow <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-11-11 08:14 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 11, 2021 at 5:52 PM [email protected]
<[email protected]> wrote:
>
> When looking into how to fix the second issue, I have a question:
>
> After changing publish_via_partition_root from false to true, the
> subcriber will fetch the partitioned table from publisher when refreshing.
>
> In subsriber side, If all the child tables of the partitioned table already
> subscribed, then we can just skip the table sync for the partitioned table. But
> if only some of the child tables(not all child tables) were already subscribed,
> should we skip the partitioned table's table sync ? I am not sure about the
> appropriate behavior here.
>
> What do you think ?
>
I'm not sure you can skip the partitioned table's table sync as you
are suggesting, because on the subscriber side, the tables are mapped
by name, so what is a partitioned table on the publisher side might
not be a partitioned table on the subscriber side (e.g. might be an
ordinary table; and similarly for the partitions) or it might be
partitioned differently to that on the publisher side. (I might be
wrong here, and I don't have a good solution, but I can see the
potential for inconsistent data resulting in this case, unless say,
the subscriber "child tables" are first truncated on the refresh, if
they are in fact partitions of the root, and then the table sync
publishes the existing data via the root)
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-12 04:11 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: Amit Kapila @ 2021-11-12 04:11 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 11, 2021 at 1:44 PM Greg Nancarrow <[email protected]> wrote:
>
> On Thu, Nov 11, 2021 at 5:52 PM [email protected]
> <[email protected]> wrote:
> >
> > When looking into how to fix the second issue, I have a question:
> >
> > After changing publish_via_partition_root from false to true, the
> > subcriber will fetch the partitioned table from publisher when refreshing.
> >
> > In subsriber side, If all the child tables of the partitioned table already
> > subscribed, then we can just skip the table sync for the partitioned table. But
> > if only some of the child tables(not all child tables) were already subscribed,
> > should we skip the partitioned table's table sync ? I am not sure about the
> > appropriate behavior here.
> >
> > What do you think ?
> >
>
> I'm not sure you can skip the partitioned table's table sync as you
> are suggesting, because on the subscriber side, the tables are mapped
> by name, so what is a partitioned table on the publisher side might
> not be a partitioned table on the subscriber side (e.g. might be an
> ordinary table; and similarly for the partitions) or it might be
> partitioned differently to that on the publisher side.
>
Sure, we don't know about that, or at least there is no such mapping
that is recorded. So, I think we should skip it even if any one of the
child table is present.
> (I might be
> wrong here, and I don't have a good solution, but I can see the
> potential for inconsistent data resulting in this case, unless say,
> the subscriber "child tables" are first truncated on the refresh, if
> they are in fact partitions of the root, and then the table sync
> publishes the existing data via the root)
>
Do you want to say current behavior for this case where data is copied
twice is okay? I think we need to find a way to handle this and
document it to set the expectations right.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-12 04:27 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-11-12 04:27 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Nov 11, 2021 at 12:22 PM [email protected]
<[email protected]> wrote:
>
> On Friday, November 5, 2021 11:20 AM Greg Nancarrow <[email protected]> wrote:
> >On Thu, Nov 4, 2021 at 7:10 PM Amit Kapila <mailto:[email protected]> wrote:
> >
> >Almost.
> >The patch does seem to solve that first problem (double publish on tablesync).
> >I used the following test (taken from [2]), and variations of it:
> >
> >However, there did still seem to be a problem, if publish_via_partition_root is then set to false; it seems that can result in
> >duplicate partition entries in the pg_publication_tables view, see below (this follows on from the test scenario given above):
> >
> >postgres=# select * from pg_publication_tables;
> > pubname | schemaname | tablename
> >---------+------------+-----------
> > pub1 | sch1 | tbl1
> > pub1 | sch3 | t1
> >(2 rows)
> >
> >postgres=# alter publication pub1 set (publish_via_partition_root=false);
> >ALTER PUBLICATION
> >postgres=# select * from pg_publication_tables;
> > pubname | schemaname | tablename
> >---------+------------+------------
> > pub1 | sch2 | tbl1_part1
> > pub1 | sch2 | tbl1_part2
> > pub1 | sch2 | tbl1_part1
> > pub1 | sch3 | t1
> >(4 rows)
> >
> >So I think the patch would need to be updated to prevent that.
>
> Thanks for testing the patch.
>
> The reason of the duplicate output is that:
> The existing function GetPublicationRelations doesn't de-duplicate the output
> oid list. So, when adding both child and parent table to the
> publication(pubviaroot = false), the pg_publication_tables view will output
> duplicate partition.
>
> Attach the fix patch.
> 0001 fix data double publish(first issue in this thread)
> 0002 fix duplicate partition in view pg_publication_tables(reported by greg when testing the 0001 patch)
>
Can we start a separate thread to discuss the 0002 patch as that
doesn't seem directly to duplicate data issues being discussed here?
Please specify the exact test in the email as that would make it
easier to understand the problem.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-16 01:56 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2021-11-16 01:56 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Friday, November 12, 2021 12:28 PM Amit Kapila <[email protected]> wrote:
>
> On Thu, Nov 11, 2021 at 12:22 PM [email protected]
> <[email protected]> wrote:
> >
> > On Friday, November 5, 2021 11:20 AM Greg Nancarrow
> <[email protected]> wrote:
> > >On Thu, Nov 4, 2021 at 7:10 PM Amit Kapila
> <mailto:[email protected]> wrote:
> > >
> > >Almost.
> > >The patch does seem to solve that first problem (double publish on
> tablesync).
> > >I used the following test (taken from [2]), and variations of it:
> > >
> > >However, there did still seem to be a problem, if
> > >publish_via_partition_root is then set to false; it seems that can result in
> duplicate partition entries in the pg_publication_tables view, see below (this
> follows on from the test scenario given above):
> > >
> > >postgres=# select * from pg_publication_tables; pubname | schemaname
> > >| tablename
> > >---------+------------+-----------
> > > pub1 | sch1 | tbl1
> > > pub1 | sch3 | t1
> > >(2 rows)
> > >
> > >postgres=# alter publication pub1 set
> > >(publish_via_partition_root=false);
> > >ALTER PUBLICATION
> > >postgres=# select * from pg_publication_tables; pubname | schemaname
> > >| tablename
> > >---------+------------+------------
> > > pub1 | sch2 | tbl1_part1
> > > pub1 | sch2 | tbl1_part2
> > > pub1 | sch2 | tbl1_part1
> > > pub1 | sch3 | t1
> > >(4 rows)
> > >
> > >So I think the patch would need to be updated to prevent that.
> >
> > Thanks for testing the patch.
> >
> > The reason of the duplicate output is that:
> > The existing function GetPublicationRelations doesn't de-duplicate the
> > output oid list. So, when adding both child and parent table to the
> > publication(pubviaroot = false), the pg_publication_tables view will
> > output duplicate partition.
> >
> > Attach the fix patch.
> > 0001 fix data double publish(first issue in this thread)
> > 0002 fix duplicate partition in view pg_publication_tables(reported by
> > greg when testing the 0001 patch)
> >
>
> Can we start a separate thread to discuss the 0002 patch as that doesn't seem
> directly to duplicate data issues being discussed here?
> Please specify the exact test in the email as that would make it easier to
> understand the problem.
Thanks for the suggestion.
I have started a new thread about this issue[1].
[1] https://www.postgresql.org/message-id/OS0PR01MB5716E97F00732B52DC2BBC2594989%40OS0PR01MB5716.jpnprd0...
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-11-29 08:51 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2021-11-29 08:51 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Thursday, November 11, 2021 2:53 PM [email protected] wrote:
> Attach the fix patch.
> 0001 fix data double publish(first issue in this thread)
In another thread[1], Amit L suggested that it'd be nice to add a testcase in
src/test/subscription/. So, attach a new version patch which add a testcase in
t/013_partition.pl.
[1] https://www.postgresql.org/message-id/CA%2BHiwqEjV%3D7iEW8hxnr73pWsDQuonDPLgsxXTYDQzDA7W9vrmw%40mail...
Best regards,
Hou zj
Attachments:
[application/octet-stream] v6-0001-Fix-double-publish-of-child-table-s-data.patch (6.3K, ../../OS0PR01MB5716303A8996B81A64EB7F6394669@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v6-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From 9c9a5b0732a87d241a28fac12883c94c10089df1 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Mon, 29 Nov 2021 16:05:03 +0800
Subject: [PATCH] Fix double publish of child table's data.
if publish_via_partition_root is true, then the child table's data will be
copied twice if adding both child and parent table to the publication. The
reason is that the subscriber will fetch the table list from publisher's
pg_publication_tables view to do the table synchronization. But the view always
show both child and parent table which cause the extra synchronization
for the child table.
Fix it by making pg_publication_tables only show parent table if both parent
and child exists in the publication.
---
src/backend/catalog/pg_publication.c | 18 ++++--------------
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 14 +++++++++++---
4 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 63579b2f82..e442d70cad 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -142,7 +142,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -163,14 +163,8 @@ filter_partitions(List *relids, List *schemarelids)
/*
* Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
*/
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -879,7 +873,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
+ if (publication->pubviaroot)
{
/*
* If the publication publishes partition changes via their
@@ -888,12 +882,8 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* tables. Otherwise, the function could return both the child
* and parent tables which could cause data of the child table
* to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
*/
- tables = filter_partitions(tables, schemarelids);
+ tables = filter_partitions(tables);
}
}
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 1feb558968..bf1484804c 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -814,6 +814,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 8fa0435c32..3b18261dcf 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -459,6 +459,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index c75a07d6b3..89e6dcd1f2 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -414,9 +414,12 @@ $node_publisher->safe_psql('postgres',
# Note: tab3_1's parent is not in the publication, in which case its
# changes are published using own identity.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +471,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-01 11:15 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: Amit Kapila @ 2021-12-01 11:15 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Mon, Nov 29, 2021 at 2:21 PM [email protected]
<[email protected]> wrote:
>
> On Thursday, November 11, 2021 2:53 PM [email protected] wrote:
> > Attach the fix patch.
> > 0001 fix data double publish(first issue in this thread)
>
> In another thread[1], Amit L suggested that it'd be nice to add a testcase in
> src/test/subscription/. So, attach a new version patch which add a testcase in
> t/013_partition.pl.
>
Thanks, your patch looks good to me. I have slightly changed the
comments and commit message in the attached.
I think we should back-patch this but I am slightly worried that if
someone is dependent on the view pg_publication_tables to return both
parent and child tables for publications that have both of those
tables and published with publish_via_partition_root as true then this
might break his usage. But OTOH, I don't see why someone would do like
that and she might face some problems like what we are trying to solve
here.
Thoughts?
--
With Regards,
Amit Kapila.
Attachments:
[application/octet-stream] v7-0001-Fix-double-publish-of-child-table-s-data.patch (7.2K, ../../CAA4eK1J=poD=yOHXZv6zs9qCgkJA0qCw62aa+WUgqpvCwMXiog@mail.gmail.com/2-v7-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From c479c0991688ad58c53bc4bb7818ff8d30a4b1e9 Mon Sep 17 00:00:00 2001
From: Amit Kapila <[email protected]>
Date: Wed, 1 Dec 2021 16:15:09 +0530
Subject: [PATCH v7] Fix double publish of child table's data.
We publish the child table data twice for publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child table as it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
---
src/backend/catalog/pg_publication.c | 41 +++++++++++--------------------
src/test/regress/expected/publication.out | 8 ++++++
src/test/regress/sql/publication.sql | 4 +++
src/test/subscription/t/013_partition.pl | 18 +++++++++++---
4 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 63579b2..96826e1 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -142,7 +142,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -161,16 +161,8 @@ filter_partitions(List *relids, List *schemarelids)
{
Oid ancestor = lfirst_oid(lc2);
- /*
- * Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
- */
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -879,22 +871,17 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
- {
- /*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude
- * partitions in favor of including the root partitioned
- * tables. Otherwise, the function could return both the child
- * and parent tables which could cause data of the child table
- * to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
- */
- tables = filter_partitions(tables, schemarelids);
- }
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
}
funcctx->user_fctx = (void *) tables;
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 1feb558..bf14848 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -814,6 +814,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 8fa0435..3b18261 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -459,6 +459,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index c75a07d..6005178 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
1.8.3.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-01 23:21 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-12-01 23:21 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Wed, Dec 1, 2021 at 10:15 PM Amit Kapila <[email protected]> wrote:
>
> Thanks, your patch looks good to me. I have slightly changed the
> comments and commit message in the attached.
>
I'd suggest tidying the patch comment a bit:
"We publish the child table's data twice for a publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child tables, since it gets both as separate tables
in the initial table list."
Also, perhaps the following additional comment (or similar) could be
added to the pg_publication_tables documentation in catalogs.sgml:
For publications of partitioned tables with publish_via_partition_root
set to true, the partitioned table itself (rather than the individual
partitions) is included in the view.
> I think we should back-patch this but I am slightly worried ...
I'd be in favor of back-patching this.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 02:33 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-12-02 02:33 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Dec 2, 2021 at 4:51 AM Greg Nancarrow <[email protected]> wrote:
>
> On Wed, Dec 1, 2021 at 10:15 PM Amit Kapila <[email protected]> wrote:
> >
>
> Also, perhaps the following additional comment (or similar) could be
> added to the pg_publication_tables documentation in catalogs.sgml:
>
> For publications of partitioned tables with publish_via_partition_root
> set to true, the partitioned table itself (rather than the individual
> partitions) is included in the view.
>
Okay, but I think it is better to add the behavior both when
publish_via_partition_root is set to true and false. As in the case of
false, it won't include the partitioned table itself.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 02:48 Greg Nancarrow <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-12-02 02:48 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Dec 2, 2021 at 1:33 PM Amit Kapila <[email protected]> wrote:
>
> > For publications of partitioned tables with publish_via_partition_root
> > set to true, the partitioned table itself (rather than the individual
> > partitions) is included in the view.
> >
>
> Okay, but I think it is better to add the behavior both when
> publish_via_partition_root is set to true and false. As in the case of
> false, it won't include the partitioned table itself.
>
If you updated my original description to say "(instead of just the
individual partitions)", it would imply the same I think.
But I don't mind if you want to explicitly state both cases to make it clear.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 03:30 Amit Langote <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: Amit Langote @ 2021-12-02 03:30 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Wed, Dec 1, 2021 at 8:15 PM Amit Kapila <[email protected]> wrote:
> On Mon, Nov 29, 2021 at 2:21 PM [email protected]
> <[email protected]> wrote:
> >
> > On Thursday, November 11, 2021 2:53 PM [email protected] wrote:
> > > Attach the fix patch.
> > > 0001 fix data double publish(first issue in this thread)
> >
> > In another thread[1], Amit L suggested that it'd be nice to add a testcase in
> > src/test/subscription/. So, attach a new version patch which add a testcase in
> > t/013_partition.pl.
> >
>
> Thanks, your patch looks good to me. I have slightly changed the
> comments and commit message in the attached.
Patch looks good to me too. I confirmed that the newly added
subscription test fails with HEAD.
> I think we should back-patch this but I am slightly worried that if
> someone is dependent on the view pg_publication_tables to return both
> parent and child tables for publications that have both of those
> tables and published with publish_via_partition_root as true then this
> might break his usage. But OTOH, I don't see why someone would do like
> that and she might face some problems like what we are trying to solve
> here.
Yeah, back-patching may not be such a bad idea.
Thank you.
--
Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 04:10 Greg Nancarrow <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Greg Nancarrow @ 2021-12-02 04:10 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Dec 2, 2021 at 1:48 PM Greg Nancarrow <[email protected]> wrote:
>
> If you updated my original description to say "(instead of just the
> individual partitions)", it would imply the same I think.
> But I don't mind if you want to explicitly state both cases to make it clear.
>
For example, something like:
For publications of partitioned tables with publish_via_partition_root
set to true, only the partitioned table (and not its partitions) is
included in the view, whereas if publish_via_partition_root is set to
false, only the individual partitions are included in the view.
Regards,
Greg Nancarrow
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 04:50 Amit Kapila <[email protected]>
parent: Greg Nancarrow <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-12-02 04:50 UTC (permalink / raw)
To: Greg Nancarrow <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>
On Thu, Dec 2, 2021 at 9:41 AM Greg Nancarrow <[email protected]> wrote:
>
> On Thu, Dec 2, 2021 at 1:48 PM Greg Nancarrow <[email protected]> wrote:
> >
> > If you updated my original description to say "(instead of just the
> > individual partitions)", it would imply the same I think.
> > But I don't mind if you want to explicitly state both cases to make it clear.
> >
>
> For example, something like:
>
> For publications of partitioned tables with publish_via_partition_root
> set to true, only the partitioned table (and not its partitions) is
> included in the view, whereas if publish_via_partition_root is set to
> false, only the individual partitions are included in the view.
>
Yeah, that sounds good to me.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-02 08:54 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2021-12-02 08:54 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Thursday, December 2, 2021 12:50 PM Amit Kapila <[email protected]> wrote:
> On Thu, Dec 2, 2021 at 9:41 AM Greg Nancarrow <[email protected]>
> wrote:
> >
> > On Thu, Dec 2, 2021 at 1:48 PM Greg Nancarrow <[email protected]>
> wrote:
> > >
> > > If you updated my original description to say "(instead of just the
> > > individual partitions)", it would imply the same I think.
> > > But I don't mind if you want to explicitly state both cases to make it clear.
> > >
> >
> > For example, something like:
> >
> > For publications of partitioned tables with publish_via_partition_root
> > set to true, only the partitioned table (and not its partitions) is
> > included in the view, whereas if publish_via_partition_root is set to
> > false, only the individual partitions are included in the view.
> >
>
> Yeah, that sounds good to me.
It looks good to me as well.
Attach the patches for (HEAD~13) which merge the suggested doc change. I
prepared the code patch and test patch separately to make it easier for committer
to confirm.
Best regards,
Hou zj
Attachments:
[application/octet-stream] HEAD-0001-Fix-double-publish-of-child-table-s-data.patch (4.2K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-HEAD-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From 31288b90d074681058a0eeec0d6bea66f14bcaf3 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 15:44:19 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table data twice for publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child table as it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
doc/src/sgml/catalogs.sgml | 10 +++++++++
src/backend/catalog/pg_publication.c | 41 ++++++++++++------------------------
2 files changed, 24 insertions(+), 27 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1d11be..b2134a9 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -11350,6 +11350,16 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
there will be a row for each eligible table.
</para>
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions are included in the
+ view.
+ </para>
+
<table>
<title><structname>pg_publication_tables</structname> Columns</title>
<tgroup cols="1">
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 63579b2..96826e1 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -142,7 +142,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -161,16 +161,8 @@ filter_partitions(List *relids, List *schemarelids)
{
Oid ancestor = lfirst_oid(lc2);
- /*
- * Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
- */
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -879,22 +871,17 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
- {
- /*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude
- * partitions in favor of including the root partitioned
- * tables. Otherwise, the function could return both the child
- * and parent tables which could cause data of the child table
- * to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
- */
- tables = filter_partitions(tables, schemarelids);
- }
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
}
funcctx->user_fctx = (void *) tables;
--
2.7.2.windows.1
[application/octet-stream] PG14-0001-Fix-double-publish-of-child-table-s-data.patch (3.7K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-PG14-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From c7236930836a2b115c72030eb718634960a2b3b9 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 14:03:28 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table data twice for publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child table as it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
doc/src/sgml/catalogs.sgml | 10 +++++++
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 688e778..8a709b6 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -11279,6 +11279,16 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
publications there will be a row for each eligible table.
</para>
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions are included in the
+ view.
+ </para>
+
<table>
<title><structname>pg_publication_tables</structname> Columns</title>
<tgroup cols="1">
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b144a3b..3660911 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables were also specified in
+ * the publication.
+ */
+static List *
+filter_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ Oid ancestor = lfirst_oid(lc2);
+
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -557,10 +596,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
+ {
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
--
2.7.2.windows.1
[application/octet-stream] PG13-0001-Fix-double-publish-of-child-table-s-data.patch (3.7K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/4-PG13-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From c7236930836a2b115c72030eb718634960a2b3b9 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 14:03:28 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table data twice for publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child table as it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
doc/src/sgml/catalogs.sgml | 10 +++++++
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 688e778..8a709b6 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -11279,6 +11279,16 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
publications there will be a row for each eligible table.
</para>
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions are included in the
+ view.
+ </para>
+
<table>
<title><structname>pg_publication_tables</structname> Columns</title>
<tgroup cols="1">
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b144a3b..3660911 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables were also specified in
+ * the publication.
+ */
+static List *
+filter_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ Oid ancestor = lfirst_oid(lc2);
+
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -557,10 +596,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
+ {
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
--
2.7.2.windows.1
[application/octet-stream] HEAD-0002-testcases.patch (4.1K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/5-HEAD-0002-testcases.patch)
download | inline diff:
From 4b7a9533d6c6167f22cd3037b5fbafca4c4f9559 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 13:25:32 +0800
Subject: [PATCH 2/2] testcases
---
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
3 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 1feb558..bf14848 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -814,6 +814,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 8fa0435..3b18261 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -459,6 +459,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index c75a07d..6005178 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.7.2.windows.1
[application/octet-stream] PG13-0002-testcases.patch (3.3K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/6-PG13-0002-testcases.patch)
download | inline diff:
From dab2aca5c5663ca8e35f3d5980ecf7b0c754cdc1 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 14:28:47 +0800
Subject: [PATCH] testcases
---
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d5..e6de145 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted1;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 3478e4d..e2a58cb 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.7.2.windows.1
[application/octet-stream] PG14-0002-testcases.patch (3.3K, ../../OS0PR01MB5716F7D44196B065449148B894699@OS0PR01MB5716.jpnprd01.prod.outlook.com/7-PG14-0002-testcases.patch)
download | inline diff:
From dab2aca5c5663ca8e35f3d5980ecf7b0c754cdc1 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 14:28:47 +0800
Subject: [PATCH] testcases
---
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d5..e6de145 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted1;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 3478e4d..e2a58cb 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-03 05:54 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2021-12-03 05:54 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Thursday, December 2, 2021 4:54 PM [email protected] <[email protected]> wrote:
> On Thursday, December 2, 2021 12:50 PM Amit Kapila
> <[email protected]> wrote:
> > On Thu, Dec 2, 2021 at 9:41 AM Greg Nancarrow <[email protected]>
> > wrote:
> > >
> > > On Thu, Dec 2, 2021 at 1:48 PM Greg Nancarrow <[email protected]>
> > wrote:
> > > >
> > > > If you updated my original description to say "(instead of just
> > > > the individual partitions)", it would imply the same I think.
> > > > But I don't mind if you want to explicitly state both cases to make it clear.
> > > >
> > >
> > > For example, something like:
> > >
> > > For publications of partitioned tables with
> > > publish_via_partition_root set to true, only the partitioned table
> > > (and not its partitions) is included in the view, whereas if
> > > publish_via_partition_root is set to false, only the individual partitions are
> included in the view.
> > >
> >
> > Yeah, that sounds good to me.
>
> It looks good to me as well.
> Attach the patches for (HEAD~13) which merge the suggested doc change. I
> prepared the code patch and test patch separately to make it easier for
> committer to confirm.
It seems we might not need to backpatch the doc change, so
attach another version which remove the doc changes from backpatch patches.
Best regards,
Hou zj
Attachments:
[application/octet-stream] HEAD-0001-Fix-double-publish-of-child-table-s-data.patch (3.4K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-HEAD-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From e3f7bba18c1ba28dcbfd0e3cfb1e1615462bc965 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 18:17:26 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table's data twice for a publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child tables, since it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
src/backend/catalog/pg_publication.c | 41 ++++++++++++------------------------
1 file changed, 14 insertions(+), 27 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 63579b2..96826e1 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -142,7 +142,7 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
* the publication.
*/
static List *
-filter_partitions(List *relids, List *schemarelids)
+filter_partitions(List *relids)
{
List *result = NIL;
ListCell *lc;
@@ -161,16 +161,8 @@ filter_partitions(List *relids, List *schemarelids)
{
Oid ancestor = lfirst_oid(lc2);
- /*
- * Check if the parent table exists in the published table list.
- *
- * XXX As of now, we do this if the partition relation or the
- * partition relation's ancestor is present in schema publication
- * relations.
- */
- if (list_member_oid(relids, ancestor) &&
- (list_member_oid(schemarelids, relid) ||
- list_member_oid(schemarelids, ancestor)))
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
{
skip = true;
break;
@@ -879,22 +871,17 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
tables = list_concat_unique_oid(relids, schemarelids);
- if (schemarelids && publication->pubviaroot)
- {
- /*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude
- * partitions in favor of including the root partitioned
- * tables. Otherwise, the function could return both the child
- * and parent tables which could cause data of the child table
- * to be double-published on the subscriber side.
- *
- * XXX As of now, we do this when a publication has associated
- * schema or for all tables publication. See
- * GetAllTablesPublicationRelations().
- */
- tables = filter_partitions(tables, schemarelids);
- }
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
}
funcctx->user_fctx = (void *) tables;
--
2.7.2.windows.1
[application/octet-stream] PG14-0001-Fix-double-publish-of-child-table-s-data.patch (2.9K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-PG14-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From fa35e5894bd66428020d8f36bdaeb57daefbcba5 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 17:22:17 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table's data twice for a publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child tables, since it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b144a3b..3660911 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables were also specified in
+ * the publication.
+ */
+static List *
+filter_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ Oid ancestor = lfirst_oid(lc2);
+
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -557,10 +596,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
+ {
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
--
2.7.2.windows.1
[application/octet-stream] PG13-0001-Fix-double-publish-of-child-table-s-data.patch (2.9K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/4-PG13-0001-Fix-double-publish-of-child-table-s-data.patch)
download | inline diff:
From fa35e5894bd66428020d8f36bdaeb57daefbcba5 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 17:22:17 +0800
Subject: [PATCH] Fix double publish of child table's data.
We publish the child table's data twice for a publication that has both
child and parent tables and is published with publish_via_partition_root
as true. This happens because subscribers will initiate synchronization
using both parent and child tables, since it gets both as separate tables
in the initial table list.
Ensure that pg_publication_tables returns only parent tables in such
cases.
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Greg Nancarrow
Discussion: https://postgr.es/m/OS0PR01MB57167F45D481F78CDC5986F794B99@OS0PR01MB5716.jpnprd01.prod.outlook.com
Backpatch-through: 13
---
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index b144a3b..3660911 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -106,6 +106,45 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
}
/*
+ * Filter out the partitions whose parent tables were also specified in
+ * the publication.
+ */
+static List *
+filter_partitions(List *relids)
+{
+ List *result = NIL;
+ ListCell *lc;
+ ListCell *lc2;
+
+ foreach(lc, relids)
+ {
+ bool skip = false;
+ List *ancestors = NIL;
+ Oid relid = lfirst_oid(lc);
+
+ if (get_rel_relispartition(relid))
+ ancestors = get_partition_ancestors(relid);
+
+ foreach(lc2, ancestors)
+ {
+ Oid ancestor = lfirst_oid(lc2);
+
+ /* Check if the parent table exists in the published table list. */
+ if (list_member_oid(relids, ancestor))
+ {
+ skip = true;
+ break;
+ }
+ }
+
+ if (!skip)
+ result = lappend_oid(result, relid);
+ }
+
+ return result;
+}
+
+/*
* Another variant of this, taking a Relation.
*/
bool
@@ -557,10 +596,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->alltables)
tables = GetAllTablesPublicationRelations(publication->pubviaroot);
else
+ {
tables = GetPublicationRelations(publication->oid,
publication->pubviaroot ?
PUBLICATION_PART_ROOT :
PUBLICATION_PART_LEAF);
+
+ /*
+ * If the publication publishes partition changes via their
+ * respective root partitioned tables, we must exclude partitions
+ * in favor of including the root partitioned tables. Otherwise,
+ * the function could return both the child and parent tables
+ * which could cause data of the child table to be
+ * double-published on the subscriber side.
+ */
+ if (publication->pubviaroot)
+ tables = filter_partitions(tables);
+ }
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
--
2.7.2.windows.1
[application/octet-stream] HEAD-0002-testcases.patch (4.1K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/5-HEAD-0002-testcases.patch)
download | inline diff:
From 4b7a9533d6c6167f22cd3037b5fbafca4c4f9559 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 13:25:32 +0800
Subject: [PATCH 2/2] testcases
---
src/test/regress/expected/publication.out | 8 ++++++++
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
3 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 1feb558..bf14848 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -814,6 +814,14 @@ SELECT * FROM pg_publication_tables;
pub | sch2 | tbl1_part1
(1 row)
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+---------+------------+-----------
+ pub | sch1 | tbl1
+(1 row)
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index 8fa0435..3b18261 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -459,6 +459,10 @@ DROP PUBLICATION pub;
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
+-- Table publication that includes both the parent table and the child table
+ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
+SELECT * FROM pg_publication_tables;
+
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index c75a07d..6005178 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.7.2.windows.1
[application/octet-stream] PG13-0002-testcases.patch (4.4K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/6-PG13-0002-testcases.patch)
download | inline diff:
From 5d0e4f64bf1c5b011a3ddea0fe2038049c316d5a Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 15:34:13 +0800
Subject: [PATCH] testcases
---
src/test/regress/expected/publication.out | 9 +++++++++
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index f27859373d..a2aca234ef 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -163,6 +163,15 @@ HINT: To enable updating the table, set REPLICA IDENTITY using ALTER TABLE.
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+-------------------+------------+----------------
+ testpub_forparted | public | testpub_parted
+(1 row)
+
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
-- Test cache invalidation FOR ALL TABLES publication
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d575b..4f2445ad11 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 3478e4db8f..e2a58cb080 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.18.4
[application/octet-stream] PG14-0002-testcases.patch (4.4K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/7-PG14-0002-testcases.patch)
download | inline diff:
From 5d0e4f64bf1c5b011a3ddea0fe2038049c316d5a Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 15:34:13 +0800
Subject: [PATCH] testcases
---
src/test/regress/expected/publication.out | 9 +++++++++
src/test/regress/sql/publication.sql | 4 ++++
src/test/subscription/t/013_partition.pl | 18 ++++++++++++++----
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index f27859373d..a2aca234ef 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -163,6 +163,15 @@ HINT: To enable updating the table, set REPLICA IDENTITY using ALTER TABLE.
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
+ pubname | schemaname | tablename
+-------------------+------------+----------------
+ testpub_forparted | public | testpub_parted
+(1 row)
+
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
-- Test cache invalidation FOR ALL TABLES publication
diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql
index e5745d575b..4f2445ad11 100644
--- a/src/test/regress/sql/publication.sql
+++ b/src/test/regress/sql/publication.sql
@@ -97,6 +97,10 @@ UPDATE testpub_parted2 SET a = 2;
ALTER PUBLICATION testpub_forparted DROP TABLE testpub_parted;
-- works again, because update is no longer replicated
UPDATE testpub_parted2 SET a = 2;
+-- publication includes both the parent table and the child table
+ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted, testpub_parted2;
+-- only parent is listed as being in publication, not the partition
+SELECT * FROM pg_publication_tables;
DROP TABLE testpub_parted1, testpub_parted2;
DROP PUBLICATION testpub_forparted, testpub_forparted1;
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 3478e4db8f..e2a58cb080 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 62;
+use Test::More tests => 63;
# setup
@@ -412,11 +412,16 @@ $node_publisher->safe_psql('postgres',
$node_publisher->safe_psql('postgres',
"ALTER PUBLICATION pub_all SET (publish_via_partition_root = true)");
# Note: tab3_1's parent is not in the publication, in which case its
-# changes are published using own identity.
+# changes are published using own identity. For tab2, even though both parent
+# and child tables are present but changes will be replicated via the parent's
+# identity and only once.
$node_publisher->safe_psql('postgres',
- "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab3_1 WITH (publish_via_partition_root = true)"
+ "CREATE PUBLICATION pub_viaroot FOR TABLE tab2, tab2_1, tab3_1 WITH (publish_via_partition_root = true)"
);
+# prepare data for the initial sync
+$node_publisher->safe_psql('postgres', "INSERT INTO tab2 VALUES (1)");
+
# subscriber 1
$node_subscriber1->safe_psql('postgres', "DROP SUBSCRIPTION sub1");
$node_subscriber1->safe_psql('postgres',
@@ -468,12 +473,17 @@ $node_subscriber1->poll_query_until('postgres', $synced_query)
$node_subscriber2->poll_query_until('postgres', $synced_query)
or die "Timed out while waiting for subscriber to synchronize data";
+# check that data is synced correctly
+$result = $node_subscriber1->safe_psql('postgres',
+ "SELECT c, a FROM tab2");
+is( $result, qq(sub1_tab2|1), 'initial data synced for pub_viaroot');
+
# insert
$node_publisher->safe_psql('postgres', "INSERT INTO tab1 VALUES (1), (0)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_1 (a) VALUES (3)");
$node_publisher->safe_psql('postgres', "INSERT INTO tab1_2 VALUES (5)");
$node_publisher->safe_psql('postgres',
- "INSERT INTO tab2 VALUES (1), (0), (3), (5)");
+ "INSERT INTO tab2 VALUES (0), (3), (5)");
$node_publisher->safe_psql('postgres',
"INSERT INTO tab3 VALUES (1), (0), (3), (5)");
--
2.18.4
[application/octet-stream] HEAD-0003-improve-the-doc-for-pg_publication_tables.patch (1.1K, ../../OS0PR01MB57165D94CB187A97128F75EA946A9@OS0PR01MB5716.jpnprd01.prod.outlook.com/8-HEAD-0003-improve-the-doc-for-pg_publication_tables.patch)
download | inline diff:
From 9d29e175216e9088fadfbb0108d7726e5c6b7785 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Thu, 2 Dec 2021 18:18:59 +0800
Subject: [PATCH] improve the doc for pg_publication_tables
---
doc/src/sgml/catalogs.sgml | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1d11be..b2134a9 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -11350,6 +11350,16 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
there will be a row for each eligible table.
</para>
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions are included in the
+ view.
+ </para>
+
<table>
<title><structname>pg_publication_tables</structname> Columns</title>
<tgroup cols="1">
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-07 12:23 vignesh C <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: vignesh C @ 2021-12-07 12:23 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Fri, Dec 3, 2021 at 11:24 AM [email protected]
<[email protected]> wrote:
>
> On Thursday, December 2, 2021 4:54 PM [email protected] <[email protected]> wrote:
> > On Thursday, December 2, 2021 12:50 PM Amit Kapila
> > <[email protected]> wrote:
> > > On Thu, Dec 2, 2021 at 9:41 AM Greg Nancarrow <[email protected]>
> > > wrote:
> > > >
> > > > On Thu, Dec 2, 2021 at 1:48 PM Greg Nancarrow <[email protected]>
> > > wrote:
> > > > >
> > > > > If you updated my original description to say "(instead of just
> > > > > the individual partitions)", it would imply the same I think.
> > > > > But I don't mind if you want to explicitly state both cases to make it clear.
> > > > >
> > > >
> > > > For example, something like:
> > > >
> > > > For publications of partitioned tables with
> > > > publish_via_partition_root set to true, only the partitioned table
> > > > (and not its partitions) is included in the view, whereas if
> > > > publish_via_partition_root is set to false, only the individual partitions are
> > included in the view.
> > > >
> > >
> > > Yeah, that sounds good to me.
> >
> > It looks good to me as well.
> > Attach the patches for (HEAD~13) which merge the suggested doc change. I
> > prepared the code patch and test patch separately to make it easier for
> > committer to confirm.
>
> It seems we might not need to backpatch the doc change, so
> attach another version which remove the doc changes from backpatch patches.
Thanks for the patches, the patch applies and the test passes in head
and the back branches. one minor suggestion:
1) Shall we change:
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions are included in the
+ view.
+ </para>
To:
+ <para>
+ For publications of partitioned tables with
+ <literal>publish_via_partition_root</literal> set to
+ <literal>true</literal>, only the partitioned table (and not its partitions)
+ is included in the view, whereas if
+ <literal>publish_via_partition_root</literal> is set to
+ <literal>false</literal>, only the individual partitions (and not the
+ partitioned table) are included in the
+ view.
+ </para>
2) Any particular reason why the code and tests are backbranched but
not the document changes?
Regards,
Vignesh
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-08 05:41 Amit Kapila <[email protected]>
parent: vignesh C <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-12-08 05:41 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Tue, Dec 7, 2021 at 5:53 PM vignesh C <[email protected]> wrote:
>
> On Fri, Dec 3, 2021 at 11:24 AM [email protected]
> <[email protected]> wrote:
> >
>
> 2) Any particular reason why the code and tests are backbranched but
> not the document changes?
>
I am not sure whether we need the doc change or not as this is not a
new feature and even if we need it as an improvement to docs, shall we
consider backpatching it? I felt that code changes are required to fix
a known issue so the case of backpatching it is clear.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-08 06:00 vignesh C <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: vignesh C @ 2021-12-08 06:00 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Wed, Dec 8, 2021 at 11:11 AM Amit Kapila <[email protected]> wrote:
>
> On Tue, Dec 7, 2021 at 5:53 PM vignesh C <[email protected]> wrote:
> >
> > On Fri, Dec 3, 2021 at 11:24 AM [email protected]
> > <[email protected]> wrote:
> > >
> >
> > 2) Any particular reason why the code and tests are backbranched but
> > not the document changes?
> >
>
> I am not sure whether we need the doc change or not as this is not a
> new feature and even if we need it as an improvement to docs, shall we
> consider backpatching it? I felt that code changes are required to fix
> a known issue so the case of backpatching it is clear.
Thanks for the clarification, I got your point. I'm fine either way
regarding the documentation change. The rest of the patch looks good
to me.
Regards,
Vignesh
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-08 06:08 Amit Kapila <[email protected]>
parent: vignesh C <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-12-08 06:08 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Wed, Dec 8, 2021 at 11:30 AM vignesh C <[email protected]> wrote:
>
> On Wed, Dec 8, 2021 at 11:11 AM Amit Kapila <[email protected]> wrote:
> >
> > On Tue, Dec 7, 2021 at 5:53 PM vignesh C <[email protected]> wrote:
> > >
> > > On Fri, Dec 3, 2021 at 11:24 AM [email protected]
> > > <[email protected]> wrote:
> > > >
> > >
> > > 2) Any particular reason why the code and tests are backbranched but
> > > not the document changes?
> > >
> >
> > I am not sure whether we need the doc change or not as this is not a
> > new feature and even if we need it as an improvement to docs, shall we
> > consider backpatching it? I felt that code changes are required to fix
> > a known issue so the case of backpatching it is clear.
>
> Thanks for the clarification, I got your point. I'm fine either way
> regarding the documentation change. The rest of the patch looks good
> to me.
>
Okay, I have also verified the code and test changes for all branches.
I'll wait for a day to see if anybody else has any comments and then
commit this.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2021-12-09 10:51 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2021-12-09 10:51 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Wed, Dec 8, 2021 at 11:38 AM Amit Kapila <[email protected]> wrote:
>
> On Wed, Dec 8, 2021 at 11:30 AM vignesh C <[email protected]> wrote:
> >
> > On Wed, Dec 8, 2021 at 11:11 AM Amit Kapila <[email protected]> wrote:
> > >
> > > On Tue, Dec 7, 2021 at 5:53 PM vignesh C <[email protected]> wrote:
> > > >
> > > > On Fri, Dec 3, 2021 at 11:24 AM [email protected]
> > > > <[email protected]> wrote:
> > > > >
> > > >
> > > > 2) Any particular reason why the code and tests are backbranched but
> > > > not the document changes?
> > > >
> > >
> > > I am not sure whether we need the doc change or not as this is not a
> > > new feature and even if we need it as an improvement to docs, shall we
> > > consider backpatching it? I felt that code changes are required to fix
> > > a known issue so the case of backpatching it is clear.
> >
> > Thanks for the clarification, I got your point. I'm fine either way
> > regarding the documentation change. The rest of the patch looks good
> > to me.
> >
>
> Okay, I have also verified the code and test changes for all branches.
> I'll wait for a day to see if anybody else has any comments and then
> commit this.
>
Pushed.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-19 07:04 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-04-19 07:04 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>; Amit Kapila <[email protected]>
> -----Original Message-----
> From: Wang, Wei/王 威 <[email protected]>
On Thursday, April 7, 2022 11:08 AM
>
> On Thur, Mar 10, 2021 at 10:08 AM [email protected] wrote:
> > Hi,
> >
> > When reviewing some logical replication related features. I noticed another
> > possible problem if the subscriber subscribes multiple publications which
> > publish parent and child table.
> >
> > For example:
> >
> > ----pub
> > create table t (a int, b int, c int) partition by range (a);
> > create table t_1 partition of t for values from (1) to (10);
> >
> > create publication pub1 for table t
> > with (PUBLISH_VIA_PARTITION_ROOT);
> > create publication pub2 for table t_1
> > with (PUBLISH_VIA_PARTITION_ROOT);
> >
> > ----sub
> > ---- prepare table t and t_1
> > CREATE SUBSCRIPTION sub CONNECTION 'port=10000 dbname=postgres'
> > PUBLICATION pub1, pub2;
> >
> > select * from pg_subscription_rel ;
> > srsubid | srrelid | srsubstate | srsublsn
> > ---------+---------+------------+-----------
> > 16391 | 16385(t) | r | 0/150D100
> > 16391 | 16388(t_1) | r | 0/150D138
> >
> > If subscribe two publications one of them publish parent table with
> > (pubviaroot=true) and another publish child table. Both the parent table and
> > child table will exist in pg_subscription_rel which also means we will do
> > initial copy for both tables.
> >
> > But after initial copy, we only publish change with the schema of the parent
> > table(t). It looks a bit inconsistent.
> >
> > Based on the document of PUBLISH_VIA_PARTITION_ROOT option. I think
> the
> > expected behavior could be we only store the top most parent(table t) in
> > pg_subscription_rel and do initial copy for it if pubviaroot is on. I haven't
> > thought about how to fix this and will investigate this later.
> Hi,
> I try to fix this bug. Attach the patch.
>
> The current HEAD get table list for one publication by invoking function
> pg_get_publication_tables. If multiple publications are subscribed, then this
> function is invoked multiple times. So option PUBLISH_VIA_PARTITION_ROOT
> works
> independently on every publication, I think it does not work correctly on
> different publications of the same subscription.
>
> So I fix this bug by the following two steps:
> First step,
> I get oids of subscribed tables by publication list. Then for tables with the
> same topmost root table, I filter them base on the option
> PUBLISH_VIA_PARTITION_ROOT(see new function filter_partitions_oids).
> After filtering, I get the final oid list.
> Second step,
> I get the required informations(nspname and relname) base on the oid list of
> first step.
Thanks for updating the patch.
I confirmed that the bug is fixed by this patch.
One suggestion is that can we simplify the code by moving the logic of checking
the ancestor into the SQL ?. For example, we could filter the outpout of
pg_publication_tables by adding A WHERE clause which checks whether the table
is a partition and if its ancestor is also in the output. I think we can also
filter the needless partition in this approach.
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-19 08:53 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: [email protected] @ 2022-04-19 08:53 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>; Amit Kapila <[email protected]>
On Tue, Apr 19, 2022 3:05 PM [email protected] <[email protected]> wrote:
>
> > -----Original Message-----
> > From: Wang, Wei/王 威 <[email protected]>
> On Thursday, April 7, 2022 11:08 AM
> >
> > On Thur, Mar 10, 2021 at 10:08 AM [email protected] wrote:
> > > Hi,
> > >
> > > When reviewing some logical replication related features. I noticed another
> > > possible problem if the subscriber subscribes multiple publications which
> > > publish parent and child table.
> > >
> > > For example:
> > >
> > > ----pub
> > > create table t (a int, b int, c int) partition by range (a);
> > > create table t_1 partition of t for values from (1) to (10);
> > >
> > > create publication pub1 for table t
> > > with (PUBLISH_VIA_PARTITION_ROOT);
> > > create publication pub2 for table t_1
> > > with (PUBLISH_VIA_PARTITION_ROOT);
> > >
> > > ----sub
> > > ---- prepare table t and t_1
> > > CREATE SUBSCRIPTION sub CONNECTION 'port=10000 dbname=postgres'
> > > PUBLICATION pub1, pub2;
> > >
> > > select * from pg_subscription_rel ;
> > > srsubid | srrelid | srsubstate | srsublsn
> > > ---------+---------+------------+-----------
> > > 16391 | 16385(t) | r | 0/150D100
> > > 16391 | 16388(t_1) | r | 0/150D138
> > >
> > > If subscribe two publications one of them publish parent table with
> > > (pubviaroot=true) and another publish child table. Both the parent table and
> > > child table will exist in pg_subscription_rel which also means we will do
> > > initial copy for both tables.
> > >
> > > But after initial copy, we only publish change with the schema of the parent
> > > table(t). It looks a bit inconsistent.
> > >
> > > Based on the document of PUBLISH_VIA_PARTITION_ROOT option. I think
> > the
> > > expected behavior could be we only store the top most parent(table t) in
> > > pg_subscription_rel and do initial copy for it if pubviaroot is on. I haven't
> > > thought about how to fix this and will investigate this later.
> > Hi,
> > I try to fix this bug. Attach the patch.
> >
> > The current HEAD get table list for one publication by invoking function
> > pg_get_publication_tables. If multiple publications are subscribed, then this
> > function is invoked multiple times. So option PUBLISH_VIA_PARTITION_ROOT
> > works
> > independently on every publication, I think it does not work correctly on
> > different publications of the same subscription.
> >
> > So I fix this bug by the following two steps:
> > First step,
> > I get oids of subscribed tables by publication list. Then for tables with the
> > same topmost root table, I filter them base on the option
> > PUBLISH_VIA_PARTITION_ROOT(see new function filter_partitions_oids).
> > After filtering, I get the final oid list.
> > Second step,
> > I get the required informations(nspname and relname) base on the oid list of
> > first step.
>
> Thanks for updating the patch.
> I confirmed that the bug is fixed by this patch.
>
> One suggestion is that can we simplify the code by moving the logic of checking
> the ancestor into the SQL ?. For example, we could filter the outpout of
> pg_publication_tables by adding A WHERE clause which checks whether the table
> is a partition and if its ancestor is also in the output. I think we can also
> filter the needless partition in this approach.
>
I agreed with you and I tried to fix this problem in a simpler way. What we want
is to exclude the partitioned table whose ancestor is also need to be
replicated, so how about implementing that by using the following SQL when
getting the table list from publisher?
SELECT DISTINCT ns.nspname, c.relname
FROM pg_catalog.pg_publication_tables t
JOIN pg_catalog.pg_namespace ns ON ns.nspname = t.schemaname
JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace = ns.oid
WHERE t.pubname IN ('p0','p2')
AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(c.oid)
WHERE relid IN ( SELECT DISTINCT (schemaname||'.'||tablename)::regclass::oid
FROM pg_catalog.pg_publication_tables t
WHERE t.pubname IN ('p0','p2') ) AND relid != c.oid));
Please find the attached patch which used this approach, I also merged the test
in Wang's patch into it.
Regards,
Shi yu
Attachments:
[application/octet-stream] v2-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (5.6K, ../../OSZPR01MB63109BE4F5521C542A75AC9BFDF29@OSZPR01MB6310.jpnprd01.prod.outlook.com/2-v2-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 8f302f5dd254853f7d0fd855ac8328a79abad38a Mon Sep 17 00:00:00 2001
From: "shiy.fnst" <[email protected]>
Date: Tue, 19 Apr 2022 11:33:33 +0800
Subject: [PATCH v2] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 26 +++++++---
src/test/subscription/t/100_bugs.pl | 69 +++++++++++++++++++++++++
2 files changed, 89 insertions(+), 6 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b94236f74d..8f63dfe047 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,20 +1759,34 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(c.oid)\n"
+ " WHERE relid IN ( SELECT DISTINCT (schemaname || '.' || tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s) ) AND relid != c.oid))", pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl
index 11ba473715..5b11e06f02 100644
--- a/src/test/subscription/t/100_bugs.pl
+++ b/src/test/subscription/t/100_bugs.pl
@@ -307,4 +307,73 @@ is( $node_subscriber->safe_psql(
$node_publisher->stop('fast');
$node_subscriber->stop('fast');
+# https://www.postgresql.org/message-id/OS0PR01MB5716DC2982CC735FDE388804940B9%40OS0PR01MB5716.jpnprd01.prod.outlook.com
+
+# The bug was that if there are two publications that publish the parent table
+# and the child table separately, and both specify the option
+# PUBLISH_VIA_PARTITION_ROOT, when subscribing to both publications with one
+# subscription, the data is replicated twice. What we expect is to be copied
+# only once.
+$node_publisher = PostgreSQL::Test::Cluster->new('publisher4');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->start;
+
+$node_subscriber = PostgreSQL::Test::Cluster->new('subscriber4');
+$node_subscriber->init(allows_streaming => 'logical');
+$node_subscriber->start;
+
+# create tables pub and sub
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE viaroot_partition(a int) PARTITION BY RANGE(a)"
+);
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE viaroot_partitioned(LIKE viaroot_partition)"
+);
+$node_publisher->safe_psql('postgres',
+ "ALTER TABLE viaroot_partition ATTACH PARTITION viaroot_partitioned DEFAULT"
+);
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE viaroot_partition(a int) PARTITION BY RANGE(a)"
+);
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE viaroot_partitioned(LIKE viaroot_partition)"
+);
+$node_subscriber->safe_psql('postgres',
+ "ALTER TABLE viaroot_partition ATTACH PARTITION viaroot_partitioned DEFAULT"
+);
+
+# insert some initial data
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO viaroot_partition (a) VALUES (1), (2), (3)"
+);
+
+# create pub/sub
+$publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION pub_viaroot1 FOR TABLE viaroot_partition WITH (PUBLISH_VIA_PARTITION_ROOT=TRUE)"
+);
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION pub_viaroot2 FOR TABLE viaroot_partitioned WITH (PUBLISH_VIA_PARTITION_ROOT=TRUE)"
+);
+$node_subscriber->safe_psql('postgres',
+ "CREATE SUBSCRIPTION sub_viaroot CONNECTION '$publisher_connstr' PUBLICATION pub_viaroot1, pub_viaroot2"
+);
+
+$node_publisher->wait_for_catchup('sub_viaroot');
+
+# Also wait for initial table sync to finish
+$node_subscriber->poll_query_until('postgres', $synced_query)
+ or die "Timed out while waiting for subscriber to synchronize data";
+
+# Check expected replicated result.
+is( $node_subscriber->safe_psql(
+ 'postgres', "SELECT * FROM viaroot_partition"),
+ qq(1
+2
+3),
+ "check initial data copy from table viaroot_partition");
+
+$node_publisher->stop('fast');
+$node_subscriber->stop('fast');
+
done_testing();
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-21 03:05 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-04-21 03:05 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; +Cc: Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>; Amit Kapila <[email protected]>
On Tue, Apr 19, 2022 4:53 PM Shi, Yu/侍 雨 <[email protected]> wrote:
> On Tue, Apr 19, 2022 3:05 PM [email protected] <[email protected]>
> wrote:
> >
> > > -----Original Message-----
> > > From: Wang, Wei/王 威 <[email protected]>
> > On Thursday, April 7, 2022 11:08 AM
> > >
> > > On Thur, Mar 10, 2021 at 10:08 AM [email protected] wrote:
> > > > Hi,
> > > >
> > > > When reviewing some logical replication related features. I noticed
> another
> > > > possible problem if the subscriber subscribes multiple publications which
> > > > publish parent and child table.
> > > >
> > > > For example:
> > > >
> > > > ----pub
> > > > create table t (a int, b int, c int) partition by range (a);
> > > > create table t_1 partition of t for values from (1) to (10);
> > > >
> > > > create publication pub1 for table t
> > > > with (PUBLISH_VIA_PARTITION_ROOT);
> > > > create publication pub2 for table t_1
> > > > with (PUBLISH_VIA_PARTITION_ROOT);
> > > >
> > > > ----sub
> > > > ---- prepare table t and t_1
> > > > CREATE SUBSCRIPTION sub CONNECTION 'port=10000 dbname=postgres'
> > > > PUBLICATION pub1, pub2;
> > > >
> > > > select * from pg_subscription_rel ;
> > > > srsubid | srrelid | srsubstate | srsublsn
> > > > ---------+---------+------------+-----------
> > > > 16391 | 16385(t) | r | 0/150D100
> > > > 16391 | 16388(t_1) | r | 0/150D138
> > > >
> > > > If subscribe two publications one of them publish parent table with
> > > > (pubviaroot=true) and another publish child table. Both the parent table
> and
> > > > child table will exist in pg_subscription_rel which also means we will do
> > > > initial copy for both tables.
> > > >
> > > > But after initial copy, we only publish change with the schema of the
> parent
> > > > table(t). It looks a bit inconsistent.
> > > >
> > > > Based on the document of PUBLISH_VIA_PARTITION_ROOT option. I think
> > > the
> > > > expected behavior could be we only store the top most parent(table t) in
> > > > pg_subscription_rel and do initial copy for it if pubviaroot is on. I haven't
> > > > thought about how to fix this and will investigate this later.
> > > Hi,
> > > I try to fix this bug. Attach the patch.
> > >
> > > The current HEAD get table list for one publication by invoking function
> > > pg_get_publication_tables. If multiple publications are subscribed, then this
> > > function is invoked multiple times. So option
> PUBLISH_VIA_PARTITION_ROOT
> > > works
> > > independently on every publication, I think it does not work correctly on
> > > different publications of the same subscription.
> > >
> > > So I fix this bug by the following two steps:
> > > First step,
> > > I get oids of subscribed tables by publication list. Then for tables with the
> > > same topmost root table, I filter them base on the option
> > > PUBLISH_VIA_PARTITION_ROOT(see new function filter_partitions_oids).
> > > After filtering, I get the final oid list.
> > > Second step,
> > > I get the required informations(nspname and relname) base on the oid list
> of
> > > first step.
> >
> > Thanks for updating the patch.
> > I confirmed that the bug is fixed by this patch.
> >
> > One suggestion is that can we simplify the code by moving the logic of
> checking
> > the ancestor into the SQL ?. For example, we could filter the outpout of
> > pg_publication_tables by adding A WHERE clause which checks whether the
> table
> > is a partition and if its ancestor is also in the output. I think we can also
> > filter the needless partition in this approach.
> >
>
> I agreed with you and I tried to fix this problem in a simpler way. What we want
> is to exclude the partitioned table whose ancestor is also need to be
> replicated, so how about implementing that by using the following SQL when
> getting the table list from publisher?
>
> SELECT DISTINCT ns.nspname, c.relname
> FROM pg_catalog.pg_publication_tables t
> JOIN pg_catalog.pg_namespace ns ON ns.nspname = t.schemaname
> JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace =
> ns.oid
> WHERE t.pubname IN ('p0','p2')
> AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1 FROM
> pg_partition_ancestors(c.oid)
> WHERE relid IN ( SELECT DISTINCT (schemaname||'.'||tablename)::regclass::oid
> FROM pg_catalog.pg_publication_tables t
> WHERE t.pubname IN ('p0','p2') ) AND relid != c.oid));
>
> Please find the attached patch which used this approach, I also merged the test
> in Wang's patch into it.
Thanks for your review and patch.
I think the approach of v2 is better than v1. It does not increase the query.
Only move the test cases from 100_bugs.pl to 013_partition.pl and simplify it.
Attach the new patch.
Regards,
Wang wei
Attachments:
[application/octet-stream] v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (4.9K, ../../OS3PR01MB6275F9606924FC06F555B1999EF49@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 0746914d6deba48b1256dc196b1e98f9aed81cef Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 20 Apr 2022 16:57:31 +0800
Subject: [PATCH v3] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 26 ++++++++++++++++++------
src/test/subscription/t/013_partition.pl | 20 +++++-------------
2 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b94236f74d..8f63dfe047 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,20 +1759,34 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(c.oid)\n"
+ " WHERE relid IN ( SELECT DISTINCT (schemaname || '.' || tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s) ) AND relid != c.oid))", pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 66e63e755e..6313d425dc 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -476,14 +476,14 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -559,11 +559,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -588,11 +583,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-21 09:41 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2022-04-21 09:41 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tue, Apr 19, 2022 at 2:23 PM [email protected]
<[email protected]> wrote:
>
> On Tue, Apr 19, 2022 3:05 PM [email protected] <[email protected]> wrote:
> >
> > One suggestion is that can we simplify the code by moving the logic of checking
> > the ancestor into the SQL ?. For example, we could filter the outpout of
> > pg_publication_tables by adding A WHERE clause which checks whether the table
> > is a partition and if its ancestor is also in the output. I think we can also
> > filter the needless partition in this approach.
> >
>
> I agreed with you and I tried to fix this problem in a simpler way. What we want
> is to exclude the partitioned table whose ancestor is also need to be
> replicated, so how about implementing that by using the following SQL when
> getting the table list from publisher?
>
> SELECT DISTINCT ns.nspname, c.relname
> FROM pg_catalog.pg_publication_tables t
> JOIN pg_catalog.pg_namespace ns ON ns.nspname = t.schemaname
> JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace = ns.oid
> WHERE t.pubname IN ('p0','p2')
> AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(c.oid)
> WHERE relid IN ( SELECT DISTINCT (schemaname||'.'||tablename)::regclass::oid
> FROM pg_catalog.pg_publication_tables t
> WHERE t.pubname IN ('p0','p2') ) AND relid != c.oid));
>
> Please find the attached patch which used this approach, I also merged the test
> in Wang's patch into it.
>
I think this will work but do we need "... relid != c.oid" at the end
of the query? If so, why? Please use an alias for
pg_partition_ancestors to make the statement understandable.
Now, this solution will work but I find this query a bit complex and
will add some overhead as we are calling pg_publication_tables
multiple times. So, I was wondering if we can have a new function
pg_get_publication_tables which takes multiple publications as input
and return the list of qualified tables? I think for back branches we
need something on the lines of what you have proposed but for HEAD we
can have a better solution.
IIRC, the column list and row filter also have some issues exactly due
to this reason, so, I would like those cases to be also mentioned here
and probably include the tests for them in the patch for HEAD.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-24 06:16 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: [email protected] @ 2022-04-24 06:16 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Thur, Apr 21, 2022 at 5:41 PM Amit Kapila <[email protected]> wrote:
>
Thanks for your comments.
> On Tue, Apr 19, 2022 at 2:23 PM [email protected] <[email protected]>
> wrote:
> >
> > On Tue, Apr 19, 2022 3:05 PM [email protected]
> <[email protected]> wrote:
> > >
> > > One suggestion is that can we simplify the code by moving the logic
> > > of checking the ancestor into the SQL ?. For example, we could
> > > filter the outpout of pg_publication_tables by adding A WHERE clause
> > > which checks whether the table is a partition and if its ancestor is
> > > also in the output. I think we can also filter the needless partition in this
> approach.
> > >
> >
> > I agreed with you and I tried to fix this problem in a simpler way.
> > What we want is to exclude the partitioned table whose ancestor is
> > also need to be replicated, so how about implementing that by using
> > the following SQL when getting the table list from publisher?
> >
> > SELECT DISTINCT ns.nspname, c.relname
> > FROM pg_catalog.pg_publication_tables t JOIN pg_catalog.pg_namespace
> > ns ON ns.nspname = t.schemaname JOIN pg_catalog.pg_class c ON
> > c.relname = t.tablename AND c.relnamespace = ns.oid WHERE t.pubname IN
> > ('p0','p2') AND (c.relispartition IS FALSE OR NOT EXISTS (SELECT 1
> > FROM pg_partition_ancestors(c.oid) WHERE relid IN ( SELECT DISTINCT
> > (schemaname||'.'||tablename)::regclass::oid
> > FROM pg_catalog.pg_publication_tables t WHERE t.pubname IN ('p0','p2')
> > ) AND relid != c.oid));
> >
> > Please find the attached patch which used this approach, I also merged
> > the test in Wang's patch into it.
> >
>
> I think this will work but do we need "... relid != c.oid" at the end of the query? If
> so, why? Please use an alias for pg_partition_ancestors to make the statement
> understandable.
I think we need this (relid != c.oid). Because when we use function
pg_partition_ancestors(c.oid), its return value not only has ancestors, but
also the input table. That is to say, when we use the table (c.oid) of the
outer query to filter in the sub-query, the table of the outer query will also
appear in the result of the sub-query.
So, I think we need this condition to prevent filtering out itself.
> Now, this solution will work but I find this query a bit complex and will add some
> overhead as we are calling pg_publication_tables multiple times. So, I was
> wondering if we can have a new function pg_get_publication_tables which
> takes multiple publications as input and return the list of qualified tables? I think
> for back branches we need something on the lines of what you have proposed
> but for HEAD we can have a better solution.
Yes, it sounds reasonable to me. Now, to fix this bug:
In the patch for HEAD, add a new function pg_get_publications_tables to get
tables info from a publications array.
In the patch for back-branch (now just share the patch for REL14), modify the
SQL to get tables info.
> IIRC, the column list and row filter also have some issues exactly due to this
> reason, so, I would like those cases to be also mentioned here and probably
> include the tests for them in the patch for HEAD.
Improve the test case about the column list and row filter to cover this bug.
Attach the new patches.[suggestions by Amit-San]
The patch for HEAD:
1. Add a new function to get tables info by a publications array.
The patch for REL14:
1. Use an alias to make the statement understandable. BTW, I adjusted the alignment.
2. Improve the test cast about the column list and row filter to cover this bug.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v1-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (12.1K, ../../OS3PR01MB6275CC45E9BC9297261865559EF99@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v1-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From a28a7d85778e6a1d66d25c27ba1e6d6489202bbe Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Sun, 24 Apr 2022 14:04:01 +0800
Subject: [PATCH v1] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we add a new function pg_get_publications_tables just like
pg_get_publication_tables but the input is the array of publications, which is
to exclude the partitioned table whose ancestor belongs to this publications
array when getting the table list.
---
src/backend/catalog/pg_publication.c | 147 +++++++++++++++++++++
src/backend/commands/subscriptioncmds.c | 17 ++-
src/include/catalog/pg_proc.dat | 5 +
src/test/subscription/t/013_partition.pl | 20 +--
src/test/subscription/t/028_row_filter.pl | 9 +-
src/test/subscription/t/031_column_list.pl | 5 +-
6 files changed, 178 insertions(+), 25 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 2631558ff1..2b90e237f5 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1154,3 +1154,150 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
SRF_RETURN_DONE(funcctx);
}
+
+/*
+ * Returns Oids of tables in publications array.
+ */
+Datum
+pg_get_publications_tables(PG_FUNCTION_ARGS)
+{
+ FuncCallContext *funcctx;
+ ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
+ Datum *elems;
+ bool *nulls;
+ int nelems;
+ List *tables_viaroot = NIL,
+ *tables = NIL,
+ *current_table = NIL;
+ int i;
+ Publication *publication;
+
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, &nulls, &nelems);
+
+ /* stuff done only on the first call of the function */
+ if (SRF_IS_FIRSTCALL())
+ {
+ MemoryContext oldcontext;
+
+ /* create a function context for cross-call persistence */
+ funcctx = SRF_FIRSTCALL_INIT();
+
+ /* switch to memory context appropriate for multiple function calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ for (i = 0; i < nelems; i++)
+ {
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), true);
+
+ if (!publication)
+ continue;
+
+ if (publication->alltables)
+ {
+ current_table = GetAllTablesPublicationRelations(publication->pubviaroot);
+ }
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_table = list_concat_unique_oid(relids, schemarelids);
+ }
+
+ /*
+ * Store the published tables separately according to pubviaroot.
+ * It is convenient for removing tables that do not need to be
+ * published later.
+ */
+ if (publication->pubviaroot)
+ tables_viaroot = list_concat_unique_oid(tables_viaroot, current_table);
+ else
+ tables = list_concat_unique_oid(tables, current_table);
+
+ /* Reset current_table */
+ list_free(current_table);
+ current_table = NIL;
+ }
+
+ /*
+ * If a partition table is published in one publication with viaroot
+ * and its parent or child table is published in another publication
+ * without viaroot, then we need to filter this whole partition tree.
+ * So, move the related partition table from tables to tables_viaroot.
+ *
+ * If option viaroot is the same for all publications, skip this part.
+ */
+ if (tables_viaroot && tables)
+ {
+ ListCell *lc;
+ ListCell *lc2;
+ List *change_tables = NIL;
+
+ foreach(lc, tables_viaroot)
+ {
+ Oid relid_viaroot = lfirst_oid(lc);
+ Oid ancestor_viaroot;
+
+ /*
+ * If the relid is not a partition table, the relid may be the
+ * root partitioned table of the relation in the tables, so
+ * check itself.
+ */
+ if (get_rel_relispartition(relid_viaroot))
+ ancestor_viaroot = llast_oid(get_partition_ancestors(relid_viaroot));
+ else
+ ancestor_viaroot = relid_viaroot;
+
+ foreach(lc2, tables)
+ {
+ Oid relid = lfirst_oid(lc2);
+ Oid ancestor;
+
+ if (get_rel_relispartition(relid))
+ ancestor = llast_oid(get_partition_ancestors(relid));
+ else
+ ancestor = relid;
+
+ if (ancestor_viaroot == ancestor)
+ {
+ tables = foreach_delete_current(tables, lc2);
+ change_tables = list_append_unique_oid(change_tables,
+ relid);
+ }
+ }
+ }
+
+ if (change_tables)
+ tables_viaroot = list_concat_unique_oid(tables_viaroot, change_tables);
+ }
+
+ if (tables_viaroot)
+ tables = list_concat_unique_oid(tables, filter_partitions(tables_viaroot));
+
+ funcctx->user_fctx = (void *) tables;
+
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ /* stuff done on every call of the function */
+ funcctx = SRF_PERCALL_SETUP();
+ tables = (List *) funcctx->user_fctx;
+
+ if (funcctx->call_cntr < list_length(tables))
+ {
+ Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+
+ SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(relid));
+ }
+
+ SRF_RETURN_DONE(funcctx);
+}
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b94236f74d..3a9e883d4f 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,17 +1759,22 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace,"
+ " LATERAL pg_get_publications_tables(array[ %s ]) gst"
+ " WHERE c.oid = gst.relid;",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6d378ff785..7f9fd025d4 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11675,6 +11675,11 @@
provolatile => 's', prorettype => 'oid', proargtypes => 'text',
proallargtypes => '{text,oid}', proargmodes => '{i,o}',
proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+{ oid => '6122', descr => 'get OIDs of tables in publication array',
+ proname => 'pg_get_publications_tables', prorows => '1000', proretset => 't',
+ provolatile => 's', prorettype => 'oid', proargtypes => 'anyarray',
+ proallargtypes => '{anyarray,oid}', proargmodes => '{i,o}',
+ proargnames => '{pubnames,relid}', prosrc => 'pg_get_publications_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 66e63e755e..6313d425dc 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -476,14 +476,14 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -559,11 +559,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -588,11 +583,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 82c4eb6ef6..21800a03b6 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -394,6 +394,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -719,8 +723,9 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is( $result, qq(16),
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is( $result, qq(16
+17),
'check replicated rows to tab_rowfilter_viaroot_part'
);
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index bdcf3e4a24..7c73913f4d 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -948,7 +948,8 @@ $node_publisher->safe_psql('postgres', qq(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -956,7 +957,7 @@ $node_publisher->safe_psql('postgres', qq(
));
$node_subscriber->safe_psql('postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.23.0.windows.1
[application/octet-stream] REL14_v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (5.4K, ../../OS3PR01MB6275CC45E9BC9297261865559EF99@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL14_v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 4156db79b9bd4b2f662c7d5a0b473333f74a45c9 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Sun, 24 Apr 2022 12:06:43 +0800
Subject: [PATCH v3] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 37 +++++++++++++++++++-----
src/test/subscription/t/013_partition.pl | 19 +++++-------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1719f04517..117d91aa52 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1477,7 +1477,8 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
ListCell *lc;
@@ -1486,10 +1487,7 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(list_length(publications) > 0);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
+ initStringInfo(&pub_names);
first = true;
foreach(lc, publications)
{
@@ -1498,14 +1496,37 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
if (first)
first = false;
else
- appendStringInfoString(&cmd, ", ");
+ appendStringInfoString(&pub_names, ", ");
- appendStringInfoString(&cmd, quote_literal_cstr(pubname));
+ appendStringInfoString(&pub_names, quote_literal_cstr(pubname));
}
- appendStringInfoChar(&cmd, ')');
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
+ initStringInfo(&cmd);
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) as relid\n"
+ " WHERE relid IN\n"
+ " (SELECT DISTINCT (schemaname || '.' || tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND relid != c.oid))\n",
+ pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 39946c735b..c57a8eb7c5 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 67;
+use Test::More tests => 65;
# setup
@@ -471,12 +471,15 @@ $node_subscriber2->safe_psql('postgres',
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
+
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations.
$node_subscriber2->safe_psql('postgres',
@@ -546,10 +549,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -574,10 +573,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.18.4
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-25 01:23 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-04-25 01:23 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Sun, Apr 24, 2022 at 2:16 PM I wrote:
> On Thur, Apr 21, 2022 at 5:41 PM Amit Kapila <[email protected]> wrote:
> > IIRC, the column list and row filter also have some issues exactly due to this
> > reason, so, I would like those cases to be also mentioned here and probably
> > include the tests for them in the patch for HEAD.
> Improve the test case about the column list and row filter to cover this bug.
Sorry, I forgot to explain why I modify the tests for row filter and column
filter. If we specify different filters on the parent and child table
respectively, this bug will make us use the wrong filter.
Like the following cases:
[row filter]
- environment in publisher-side.
create table t (a int) partition by range (a);
create table t_1 partition of t default;
create publication pub1 for table t where (a<=10) with (PUBLISH_VIA_PARTITION_ROOT=true);
create publication pub2 for table t_1 where (a>10) with (PUBLISH_VIA_PARTITION_ROOT=true);
insert into t values (9),(11);
- environment in subscriber-side.
create table t (a int) partition by range (a);
create table t_1 partition of t default;
create subscription sub connection 'dbname=postgres user=postgres' publication pub1,pub2;
When we execute the following SQL in subscriber-side, what we expect should be:
select * from t;
a
---
9
(1 row)
but the HEAD is:
a
----
9
11
(2 rows)
[column filter]
- environment in publisher-side.
create table t (a int primary key, b int, c int) partition by range (a);
create table t_1 partition of t default;
create publication pub1 for table t(a, b) with (PUBLISH_VIA_PARTITION_ROOT=true);
create publication pub2 for table t_1(a, c) with (PUBLISH_VIA_PARTITION_ROOT=true);
insert into t values (1,1,1);
- environment in subscriber-side.
create table t (a int, b int, c int) partition by range (a);
create table t_1 partition of t default;
create subscription sub connection 'dbname=postgres user=postgres' publication pub1,pub2;
When we execute the following SQL in subscriber-side, what we expect should be:
select * from t;
a | b | c
---+---+---
1 | 1 |
(1 row)
but the HEAD is:
a | b | c
---+---+---
1 | 1 |
1 | | 1
(2 rows)
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-04-28 01:22 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-04-28 01:22 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Sun, Apr 24, 2022 2:16 PM Wang, Wei/王 威 <[email protected]> wrote:
>
> Attach the new patches.[suggestions by Amit-San]
> The patch for HEAD:
> 1. Add a new function to get tables info by a publications array.
> The patch for REL14:
> 1. Use an alias to make the statement understandable. BTW, I adjusted the
> alignment.
> 2. Improve the test cast about the column list and row filter to cover this bug.
>
Thanks for your patches.
Here's a comment on the patch for REL14.
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) as relid\n"
+ " WHERE relid IN\n"
+ " (SELECT DISTINCT (schemaname || '.' || tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND relid != c.oid))\n",
+ pub_names.data, pub_names.data);
I think we can use an alias like 'pa' for pg_partition_ancestors, and modify the SQL as follows.
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) pa\n"
+ " WHERE pa.relid IN\n"
+ " (SELECT DISTINCT (t.schemaname || '.' || t.tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND pa.relid != c.oid))\n",
+ pub_names.data, pub_names.data);
Regards,
Shi yu
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-09 01:51 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-09 01:51 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tue, Apr 28, 2022 9:22 AM Shi, Yu/侍 雨 <[email protected]> wrote:
> Thanks for your patches.
>
> Here's a comment on the patch for REL14.
Thanks for your comments.
> + appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
> + " FROM
> pg_catalog.pg_publication_tables t\n"
> + " JOIN pg_catalog.pg_namespace
> ns\n"
> + " ON ns.nspname =
> t.schemaname\n"
> + " JOIN pg_catalog.pg_class c\n"
> + " ON c.relname = t.tablename AND
> c.relnamespace = ns.oid\n"
> + " WHERE t.pubname IN (%s)\n"
> + " AND (c.relispartition IS FALSE\n"
> + " OR NOT EXISTS\n"
> + " ( SELECT 1 FROM
> pg_partition_ancestors(c.oid) as relid\n"
> + " WHERE relid IN\n"
> + " (SELECT DISTINCT (schemaname
> || '.' || tablename)::regclass::oid\n"
> + " FROM
> pg_catalog.pg_publication_tables t\n"
> + " WHERE t.pubname IN (%s))\n"
> + " AND relid != c.oid))\n",
> + pub_names.data, pub_names.data);
>
> I think we can use an alias like 'pa' for pg_partition_ancestors, and modify the
> SQL as follows.
>
> + appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
> + " FROM
> pg_catalog.pg_publication_tables t\n"
> + " JOIN pg_catalog.pg_namespace
> ns\n"
> + " ON ns.nspname =
> t.schemaname\n"
> + " JOIN pg_catalog.pg_class c\n"
> + " ON c.relname = t.tablename AND
> c.relnamespace = ns.oid\n"
> + " WHERE t.pubname IN (%s)\n"
> + " AND (c.relispartition IS FALSE\n"
> + " OR NOT EXISTS\n"
> + " ( SELECT 1 FROM
> pg_partition_ancestors(c.oid) pa\n"
> + " WHERE pa.relid IN\n"
> + " (SELECT DISTINCT
> (t.schemaname || '.' || t.tablename)::regclass::oid\n"
> + " FROM
> pg_catalog.pg_publication_tables t\n"
> + " WHERE t.pubname IN (%s))\n"
> + " AND pa.relid != c.oid))\n",
> + pub_names.data, pub_names.data);
Fix it.
In addition, I try to modify the approach for the HEAD.
I enhance the API of function pg_get_publication_tables. Change the parameter
type from 'text' to 'any'. Then we can use this function to get tables from one
publication or an array of publications. Any thoughts on this approach?
Attach new patches.
The patch for HEAD:
1. Modify the approach. Enhance the API of function pg_get_publication_tables to
handle one publication or an array of publications.
The patch for REL14:
1. Improve the table sync SQL. [suggestions by Shi yu]
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v2-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (14.6K, ../../OS3PR01MB62751C2FA25E23E24AB6980B9EC69@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v2-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From e8d9cf1b5b3cbfc590a5f1853aacac905823bb71 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Thu, 28 Apr 2022 13:37:19 +0800
Subject: [PATCH v2] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we extended the API of the function pg_get_publication_tables.
(Change the parameter type from text to any).
So, the function pg_get_publication_tables could also receive the array of
publications. And then, if we specify option viaroot, we could exclude the
partitioned table whose ancestor belongs to this publications array when
getting the table list.
---
src/backend/catalog/pg_publication.c | 169 ++++++++++++++++-----
src/backend/commands/subscriptioncmds.c | 17 ++-
src/include/catalog/pg_proc.dat | 6 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 20 +--
src/test/subscription/t/028_row_filter.pl | 9 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 164 insertions(+), 64 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 2631558ff1..ada93ac8db 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,20 +1077,55 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns Oids of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ Oid typeid;
+ ArrayType *arr;
+ Datum *elems;
+ bool *nulls;
+ int nelems;
+ char *pubname;
+ List *tables_viaroot = NIL,
+ *tables = NIL,
+ *current_table = NIL;
+
+ typeid = get_fn_expr_argtype(fcinfo->flinfo, 0);
+
+ switch (typeid)
+ {
+ case TEXTARRAYOID:
+ /* get Oids of tables in publications array */
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, &nulls, &nelems);
+ break;
+ case TEXTOID:
+ case NAMEOID:
+ case UNKNOWNOID:
+ /* get Oids of tables in a publication */
+ elems = (Datum *) palloc(sizeof(Datum));
+ if (typeid == TEXTOID)
+ elems[0] = PG_GETARG_DATUM(0);
+ else
+ elems[0] = CStringGetTextDatum(PG_GETARG_POINTER(0));
+ nelems = 1;
+ break;
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("do not support this argument type")));
+ }
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
+ Publication *publication;
+ int i;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1098,44 +1133,108 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), true);
+
+ if (!publication)
+ continue;
+
+ /*
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
+ */
+ if (publication->alltables)
+ {
+ current_table = GetAllTablesPublicationRelations(publication->pubviaroot);
+ }
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_table = list_concat_unique_oid(relids, schemarelids);
+ }
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Store the published tables separately according to pubviaroot.
+ * It is convenient for removing tables that do not need to be
+ * published later.
*/
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ tables_viaroot = list_concat_unique_oid(tables_viaroot, current_table);
+ else
+ tables = list_concat_unique_oid(tables, current_table);
+
+ /* Reset current_table */
+ list_free(current_table);
+ current_table = NIL;
}
+ /*
+ * If a partition table is published in a publication with viaroot,
+ * and its parent or child table is published in another publication
+ * without viaroot. Then we need to move the parent or child table
+ * from tables to tables_viaroot.
+ *
+ * If all publication(s)'s viaroot are the same, then skip this part.
+ */
+ if (tables_viaroot && tables)
+ {
+ ListCell *lc;
+ ListCell *lc2;
+ List *change_tables = NIL;
+
+ foreach(lc, tables_viaroot)
+ {
+ Oid relid_viaroot = lfirst_oid(lc);
+ Oid ancestor_viaroot;
+
+ /*
+ * If the relid is not a partition table, the relid may be the
+ * root partitioned table of the relation in the tables, so
+ * check itself.
+ */
+ if (get_rel_relispartition(relid_viaroot))
+ ancestor_viaroot = llast_oid(get_partition_ancestors(relid_viaroot));
+ else
+ ancestor_viaroot = relid_viaroot;
+
+ foreach(lc2, tables)
+ {
+ Oid relid = lfirst_oid(lc2);
+ Oid ancestor;
+
+ if (get_rel_relispartition(relid))
+ ancestor = llast_oid(get_partition_ancestors(relid));
+ else
+ ancestor = relid;
+
+ if (ancestor_viaroot == ancestor)
+ {
+ tables = foreach_delete_current(tables, lc2);
+ change_tables = list_append_unique_oid(change_tables,
+ relid);
+ }
+ }
+ }
+
+ if (change_tables)
+ tables_viaroot = list_concat_unique_oid(tables_viaroot, change_tables);
+ }
+
+ if (tables_viaroot)
+ tables = list_concat_unique_oid(tables, filter_partitions(tables_viaroot));
+
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b94236f74d..851eabf3f1 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,17 +1759,22 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace,"
+ " LATERAL pg_get_publication_tables(array[ %s ]) gst"
+ " WHERE c.oid = gst.relid;",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 6d378ff785..2cd3712365 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11670,10 +11670,10 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
+ provolatile => 's', prorettype => 'oid', proargtypes => 'any',
+ proallargtypes => '{any,oid}', proargmodes => '{i,o}',
proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8315..35f1f4241d 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1439,7 +1439,7 @@ pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
c.relname AS tablename
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables(p.pubname) gpt(relid),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 66e63e755e..6313d425dc 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -476,14 +476,14 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -559,11 +559,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -588,11 +583,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 82c4eb6ef6..21800a03b6 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -394,6 +394,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -719,8 +723,9 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is( $result, qq(16),
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is( $result, qq(16
+17),
'check replicated rows to tab_rowfilter_viaroot_part'
);
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index bdcf3e4a24..7c73913f4d 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -948,7 +948,8 @@ $node_publisher->safe_psql('postgres', qq(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -956,7 +957,7 @@ $node_publisher->safe_psql('postgres', qq(
));
$node_subscriber->safe_psql('postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.18.4
[application/octet-stream] REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (5.4K, ../../OS3PR01MB62751C2FA25E23E24AB6980B9EC69@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From d95019cbaaee049ee3535ef0e6ab455fb2419043 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Thu, 28 Apr 2022 11:28:35 +0800
Subject: [PATCH v4] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 37 +++++++++++++++++++-----
src/test/subscription/t/013_partition.pl | 19 +++++-------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1719f04517..6ab6f249d0 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1477,7 +1477,8 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
ListCell *lc;
@@ -1486,10 +1487,7 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(list_length(publications) > 0);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
+ initStringInfo(&pub_names);
first = true;
foreach(lc, publications)
{
@@ -1498,14 +1496,37 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
if (first)
first = false;
else
- appendStringInfoString(&cmd, ", ");
+ appendStringInfoString(&pub_names, ", ");
- appendStringInfoString(&cmd, quote_literal_cstr(pubname));
+ appendStringInfoString(&pub_names, quote_literal_cstr(pubname));
}
- appendStringInfoChar(&cmd, ')');
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
+ initStringInfo(&cmd);
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) as pa\n"
+ " WHERE pa.relid IN\n"
+ " (SELECT DISTINCT (t.schemaname || '.' || t.tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND pa.relid != c.oid))\n",
+ pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 39946c735b..c57a8eb7c5 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 67;
+use Test::More tests => 65;
# setup
@@ -471,12 +471,15 @@ $node_subscriber2->safe_psql('postgres',
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
+
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations.
$node_subscriber2->safe_psql('postgres',
@@ -546,10 +549,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -574,10 +573,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-11 02:32 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-11 02:32 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Monday, May 9, 2022 10:51 AM [email protected] <[email protected]> wrote:
> Attach new patches.
> The patch for HEAD:
> 1. Modify the approach. Enhance the API of function
> pg_get_publication_tables to handle one publication or an array of
> publications.
> The patch for REL14:
> 1. Improve the table sync SQL. [suggestions by Shi yu]
Hi, thank you for updating the patch !
Minor comments on your patch for HEAD v2.
(1) commit message sentence
I suggest below sentence.
Kindly change from
"... when subscribing to both publications using one subscription, the data is replicated
twice in inital copy"
to "subscribing to both publications from one subscription causes initial copy twice".
(2) unused variable
pg_publication.c: In function ‘pg_get_publication_tables’:
pg_publication.c:1091:11: warning: unused variable ‘pubname’ [-Wunused-variable]
char *pubname;
We can remove this.
(3) free of allocated memory
In the pg_get_publication_tables(),
we don't free 'elems'. Don't we need it ?
(4) some coding alignments
4-1.
+ List *tables_viaroot = NIL,
...
+ *current_table = NIL;
I suggest we can put some variables
into the condition for the first time call of this function,
like tables_viaroot and current_table.
When you agree, kindly change it.
4-2.
+ /*
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
+ */
+ if (publication->alltables)
+ {
+ current_table = GetAllTablesPublicationRelations(publication->pubviaroot);
+ }
This is not related to the change itself and now
we are inheriting the previous curly brackets, but
I think there's no harm in removing it, since it's only for one statement.
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-12 01:47 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-12 01:47 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wednesday, May 11, 2022 11:33 AM I wrote:
> On Monday, May 9, 2022 10:51 AM [email protected]
> <[email protected]> wrote:
> > Attach new patches.
> > The patch for HEAD:
> > 1. Modify the approach. Enhance the API of function
> > pg_get_publication_tables to handle one publication or an array of
> > publications.
> > The patch for REL14:
> > 1. Improve the table sync SQL. [suggestions by Shi yu]
> Hi, thank you for updating the patch !
>
> Minor comments on your patch for HEAD v2.
>
> (1) commit message sentence
>
> I suggest below sentence.
>
> Kindly change from
> "... when subscribing to both publications using one subscription, the data is
> replicated twice in inital copy"
> to "subscribing to both publications from one subscription causes initial copy
> twice".
>
> (2) unused variable
>
> pg_publication.c: In function ‘pg_get_publication_tables’:
> pg_publication.c:1091:11: warning: unused variable ‘pubname’
> [-Wunused-variable]
> char *pubname;
>
> We can remove this.
>
> (3) free of allocated memory
>
> In the pg_get_publication_tables(),
> we don't free 'elems'. Don't we need it ?
>
> (4) some coding alignments
>
> 4-1.
>
> + List *tables_viaroot = NIL,
> ...
> + *current_table = NIL;
>
> I suggest we can put some variables
> into the condition for the first time call of this function, like tables_viaroot and
> current_table.
> When you agree, kindly change it.
>
> 4-2.
>
> + /*
> + * Publications support partitioned tables, although
> all changes
> + * are replicated using leaf partition identity and
> schema, so we
> + * only need those.
> + */
> + if (publication->alltables)
> + {
> + current_table =
> GetAllTablesPublicationRelations(publication->pubviaroot);
> + }
>
> This is not related to the change itself and now we are inheriting the previous
> curly brackets, but I think there's no harm in removing it, since it's only for one
> statement.
Hi,
One more thing I'd like to add is that
we don't hit the below code by tests.
In the HEAD v2, we add a new filtering logic in pg_get_publication_tables.
Although I'm not sure if this is related to the bug fix itself,
when we want to include it in this patch, then
I feel it's better to add some simple test for this part,
to cover all the new main paths and check if
new logic works correctly.
+ /*
+ * If a partition table is published in a publication with viaroot,
+ * and its parent or child table is published in another publication
+ * without viaroot. Then we need to move the parent or child table
+ * from tables to tables_viaroot.
+ *
+ * If all publication(s)'s viaroot are the same, then skip this part.
+ */
....
if (ancestor_viaroot == ancestor)
+ {
+ tables = foreach_delete_current(tables, lc2);
+ change_tables = list_append_unique_oid(change_tables,
+ relid);
+ }
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-13 02:02 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-13 02:02 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Thur, May 12, 2022 9:48 AM [email protected] <[email protected]> wrote:
> On Wednesday, May 11, 2022 11:33 AM I wrote:
> > On Monday, May 9, 2022 10:51 AM [email protected]
> > <[email protected]> wrote:
> > > Attach new patches.
> > > The patch for HEAD:
> > > 1. Modify the approach. Enhance the API of function
> > > pg_get_publication_tables to handle one publication or an array of
> > > publications.
> > > The patch for REL14:
> > > 1. Improve the table sync SQL. [suggestions by Shi yu]
> > Hi, thank you for updating the patch !
> >
> > Minor comments on your patch for HEAD v2.
Thanks for your comments.
> > (1) commit message sentence
> >
> > I suggest below sentence.
> >
> > Kindly change from
> > "... when subscribing to both publications using one subscription, the data is
> > replicated twice in inital copy"
> > to "subscribing to both publications from one subscription causes initial copy
> > twice".
Improve it according to your suggestion.
> > (2) unused variable
> >
> > pg_publication.c: In function ‘pg_get_publication_tables’:
> > pg_publication.c:1091:11: warning: unused variable ‘pubname’
> > [-Wunused-variable]
> > char *pubname;
> >
> > We can remove this.
Fix it.
> > (3) free of allocated memory
> >
> > In the pg_get_publication_tables(),
> > we don't free 'elems'. Don't we need it ?
Improve it according to your suggestion. Free 'elems'.
> > (4) some coding alignments
> >
> > 4-1.
> >
> > + List *tables_viaroot = NIL,
> > ...
> > + *current_table = NIL;
> >
> > I suggest we can put some variables
> > into the condition for the first time call of this function, like tables_viaroot and
> > current_table.
> > When you agree, kindly change it.
Improve these according to your suggestions.
Also, I put the code for getting publication(s) into the condition for the
first time call of this function.
> > 4-2.
> >
> > + /*
> > + * Publications support partitioned tables, although
> > all changes
> > + * are replicated using leaf partition identity and
> > schema, so we
> > + * only need those.
> > + */
> > + if (publication->alltables)
> > + {
> > + current_table =
> > GetAllTablesPublicationRelations(publication->pubviaroot);
> > + }
> >
> > This is not related to the change itself and now we are inheriting the previous
> > curly brackets, but I think there's no harm in removing it, since it's only for one
> > statement.
Improve these according to your suggestions.
> Hi,
>
> One more thing I'd like to add is that
> we don't hit the below code by tests.
> In the HEAD v2, we add a new filtering logic in pg_get_publication_tables.
> Although I'm not sure if this is related to the bug fix itself,
> when we want to include it in this patch, then
> I feel it's better to add some simple test for this part,
> to cover all the new main paths and check if
> new logic works correctly.
>
>
> + /*
> + * If a partition table is published in a publication with viaroot,
> + * and its parent or child table is published in another publication
> + * without viaroot. Then we need to move the parent or child table
> + * from tables to tables_viaroot.
> + *
> + * If all publication(s)'s viaroot are the same, then skip this part.
> + */
>
> ....
> if (ancestor_viaroot == ancestor)
> + {
> + tables = foreach_delete_current(tables, lc2);
> + change_tables =
> list_append_unique_oid(change_tables,
> + relid);
> + }
Yes, I agree.
But when I was adding the test, I found we could improve this part.
So, I removed this part of the code.
Also rebase it because the change in HEAD(23e7b38).
Attach the patches.(Only changed the patch for HEAD.).
1. Improve the commit message. [suggestions by Osumi-san]
2. Improve coding alignments and the usage for SRFs. [suggestions by Osumi-san and I]
3. Simplify the modifications in function pg_get_publication_tables.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (12.9K, ../../OS3PR01MB6275E64C5C8BD561FD6E57359ECA9@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v3-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From d1f860659a02774eea514cfd65095df591407325 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Fri, 13 May 2022 09:27:41 +0800
Subject: [PATCH v3] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
To fix this, we extended the API of the function pg_get_publication_tables.
(Change the parameter type from text to any).
So, the function pg_get_publication_tables could also receive the array of
publications. And then, if we specify option viaroot, we could exclude the
partitioned table whose ancestor belongs to this publications array when
getting the table list.
---
src/backend/catalog/pg_publication.c | 98 ++++++++++++++++------
src/backend/commands/subscriptioncmds.c | 17 ++--
src/include/catalog/pg_proc.dat | 6 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 ++---
src/test/subscription/t/028_row_filter.pl | 11 ++-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 104 insertions(+), 54 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb279..b2d91c67a3 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,20 +1077,24 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns Oids of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
+ Oid typeid;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems;
+ Publication *publication;
+ int i;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1098,31 +1102,64 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* get publication(s) */
+ typeid = get_fn_expr_argtype(fcinfo->flinfo, 0);
+ switch (typeid)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ case TEXTARRAYOID:
+ /* get Oids of tables in publications array */
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+ break;
+ case TEXTOID:
+ case NAMEOID:
+ case UNKNOWNOID:
+ /* get Oids of tables in a publication */
+ elems = (Datum *) palloc(sizeof(Datum));
+ if (typeid == TEXTOID)
+ elems[0] = PG_GETARG_DATUM(0);
+ else
+ elems[0] = CStringGetTextDatum(PG_GETARG_POINTER(0));
+ nelems = 1;
+ break;
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("do not support this argument type")));
}
- else
+
+ /* get Oids of tables from each publication */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), true);
+
+ if (!publication)
+ continue;
+
+ /*
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
+ */
+ if (publication->alltables)
+ tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ tables = list_concat_unique_oid(tables, relids);
+ tables = list_concat_unique_oid(tables, schemarelids);
+ }
/*
* If the publication publishes partition changes via their
@@ -1135,6 +1172,15 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->pubviaroot)
tables = filter_partitions(tables);
}
+ pfree(elems);
+
+ /*
+ * We need an additional filter for this case : A partition table is
+ * published in a publication with viaroot, and its parent or child
+ * table is published in another publication without viaroot. In this
+ * case, we should publish only parent table.
+ */
+ tables = filter_partitions(tables);
funcctx->user_fctx = (void *) tables;
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa426..62f2178626 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,17 +1759,22 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace,"
+ " LATERAL pg_get_publication_tables(array[ %s ]) gst"
+ " WHERE c.oid = gst.relid;",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f00a..371f8abe6c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,10 +11673,10 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
+ provolatile => 's', prorettype => 'oid', proargtypes => 'any',
+ proallargtypes => '{any,oid}', proargmodes => '{i,o}',
proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8315..35f1f4241d 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1439,7 +1439,7 @@ pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
c.relname AS tablename
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables(p.pubname) gpt(relid),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index e7f4a94f19..549252971c 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 0dc0a6d10f..c72a98bed7 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -718,8 +722,11 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e11f3..6e974aa601 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -1040,7 +1040,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -1049,7 +1050,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.18.4
[application/octet-stream] REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (5.4K, ../../OS3PR01MB6275E64C5C8BD561FD6E57359ECA9@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From d95019cbaaee049ee3535ef0e6ab455fb2419043 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Thu, 28 Apr 2022 11:28:35 +0800
Subject: [PATCH v4] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 37 +++++++++++++++++++-----
src/test/subscription/t/013_partition.pl | 19 +++++-------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1719f04517..6ab6f249d0 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1477,7 +1477,8 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
ListCell *lc;
@@ -1486,10 +1487,7 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(list_length(publications) > 0);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
+ initStringInfo(&pub_names);
first = true;
foreach(lc, publications)
{
@@ -1498,14 +1496,37 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
if (first)
first = false;
else
- appendStringInfoString(&cmd, ", ");
+ appendStringInfoString(&pub_names, ", ");
- appendStringInfoString(&cmd, quote_literal_cstr(pubname));
+ appendStringInfoString(&pub_names, quote_literal_cstr(pubname));
}
- appendStringInfoChar(&cmd, ')');
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
+ initStringInfo(&cmd);
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) as pa\n"
+ " WHERE pa.relid IN\n"
+ " (SELECT DISTINCT (t.schemaname || '.' || t.tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND pa.relid != c.oid))\n",
+ pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 39946c735b..c57a8eb7c5 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 67;
+use Test::More tests => 65;
# setup
@@ -471,12 +471,15 @@ $node_subscriber2->safe_psql('postgres',
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
+
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations.
$node_subscriber2->safe_psql('postgres',
@@ -546,10 +549,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -574,10 +573,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-13 05:58 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2022-05-13 05:58 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, May 13, 2022 at 7:32 AM [email protected]
<[email protected]> wrote:
>
> Attach the patches.(Only changed the patch for HEAD.).
>
Few comments:
=============
1.
@@ -1135,6 +1172,15 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (publication->pubviaroot)
tables = filter_partitions(tables);
}
+ pfree(elems);
+
+ /*
+ * We need an additional filter for this case : A partition table is
+ * published in a publication with viaroot, and its parent or child
+ * table is published in another publication without viaroot. In this
+ * case, we should publish only parent table.
+ */
+ tables = filter_partitions(tables);
Do we need to filter partitions twice? Can't we check if any of the
publications has 'pubviaroot' option set, if so, call
filter_partitions at the end?
2. " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace,"
+ " LATERAL pg_get_publication_tables(array[ %s ]) gst"
Here, it is better to have an alias name as gpt.
3.
}
+ pfree(elems);
+
An extra line between these two lines makes it looks slightly better.
4. Not able to apply patch cleanly.
patching file src/test/subscription/t/013_partition.pl
Hunk #1 FAILED at 477.
Hunk #2 FAILED at 556.
Hunk #3 FAILED at 584.
3 out of 3 hunks FAILED -- saving rejects to file
src/test/subscription/t/013_partition.pl.rej
patching file src/test/subscription/t/028_row_filter.pl
Hunk #1 succeeded at 394 (offset 1 line).
Hunk #2 FAILED at 722.
1 out of 2 hunks FAILED -- saving rejects to file
src/test/subscription/t/028_row_filter.pl.rej
patching file src/test/subscription/t/031_column_list.pl
Hunk #1 succeeded at 948 (offset -92 lines).
Hunk #2 FAILED at 1050.
1 out of 2 hunks FAILED -- saving rejects to file
src/test/subscription/t/031_column_list.pl.rej
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-13 09:41 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: [email protected] @ 2022-05-13 09:41 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, May 13, 2022 1:59 PM Amit Kapila <[email protected]> wrote:
> On Fri, May 13, 2022 at 7:32 AM [email protected]
> <[email protected]> wrote:
> >
> > Attach the patches.(Only changed the patch for HEAD.).
> >
>
> Few comments:
> =============
Thanks for your comments.
> 1.
> @@ -1135,6 +1172,15 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
> if (publication->pubviaroot)
> tables = filter_partitions(tables);
> }
> + pfree(elems);
> +
> + /*
> + * We need an additional filter for this case : A partition table is
> + * published in a publication with viaroot, and its parent or child
> + * table is published in another publication without viaroot. In this
> + * case, we should publish only parent table.
> + */
> + tables = filter_partitions(tables);
>
> Do we need to filter partitions twice? Can't we check if any of the publications
> has 'pubviaroot' option set, if so, call filter_partitions at the end?
Improve it according to your suggestion.
> 2. " FROM pg_class c JOIN pg_namespace n"
> + " ON n.oid = c.relnamespace,"
> + " LATERAL pg_get_publication_tables(array[ %s ]) gst"
>
> Here, it is better to have an alias name as gpt.
Improve it according to your suggestion.
> 3.
> }
> + pfree(elems);
> +
>
> An extra line between these two lines makes it looks slightly better.
Improve it according to your suggestion.
> 4. Not able to apply patch cleanly.
> patching file src/test/subscription/t/013_partition.pl
> Hunk #1 FAILED at 477.
> Hunk #2 FAILED at 556.
> Hunk #3 FAILED at 584.
> 3 out of 3 hunks FAILED -- saving rejects to file
> src/test/subscription/t/013_partition.pl.rej
> patching file src/test/subscription/t/028_row_filter.pl
> Hunk #1 succeeded at 394 (offset 1 line).
> Hunk #2 FAILED at 722.
> 1 out of 2 hunks FAILED -- saving rejects to file
> src/test/subscription/t/028_row_filter.pl.rej
> patching file src/test/subscription/t/031_column_list.pl
> Hunk #1 succeeded at 948 (offset -92 lines).
> Hunk #2 FAILED at 1050.
> 1 out of 2 hunks FAILED -- saving rejects to file
> src/test/subscription/t/031_column_list.pl.rej
New patch could apply patch cleanly now.
Attach the patches.(Only changed the patch for HEAD.).
1. Optimize the code. Reduce calls to function filter_partitions. [suggestions by Amit-san]
2. Improve the alias name in SQL. [suggestions by Amit-san]
3. Improve coding alignments. [suggestions by Amit-san]
4. Do some optimizations for list Concatenate.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (13.4K, ../../OSZPR01MB62789D224DEEBA60924301599ECA9@OSZPR01MB6278.jpnprd01.prod.outlook.com/2-HEAD_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 6646c961c06e547449bb59d78e24db61d03e9dc6 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Fri, 13 May 2022 09:27:41 +0800
Subject: [PATCH v4] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
To fix this, we extended the API of the function pg_get_publication_tables.
(Change the parameter type from text to any).
So, the function pg_get_publication_tables could also receive the array of
publications. And then, if we specify option viaroot, we could exclude the
partitioned table whose ancestor belongs to this publications array when
getting the table list.
---
src/backend/catalog/pg_publication.c | 113 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 17 ++--
src/include/catalog/pg_proc.dat | 6 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 +---
src/test/subscription/t/028_row_filter.pl | 11 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 112 insertions(+), 61 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb279..03a4d162fe 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,20 +1077,25 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns Oids of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
+ Oid typeid;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems;
+ Publication *publication;
+ int i;
+ bool viaroot = false;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1098,44 +1103,86 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* get publication(s) */
+ typeid = get_fn_expr_argtype(fcinfo->flinfo, 0);
+ switch (typeid)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ case TEXTARRAYOID:
+ /* get Oids of tables in publications array */
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+ break;
+ case TEXTOID:
+ case NAMEOID:
+ case UNKNOWNOID:
+ /* get Oids of tables in a publication */
+ elems = (Datum *) palloc(sizeof(Datum));
+ if (typeid == TEXTOID)
+ elems[0] = PG_GETARG_DATUM(0);
+ else
+ elems[0] = CStringGetTextDatum(PG_GETARG_POINTER(0));
+ nelems = 1;
+ break;
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("do not support this argument type")));
}
- else
+
+ /* get Oids of tables from each publication */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), true);
+
+ if (!publication)
+ continue;
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
*/
+ if (publication->alltables)
+ tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ tables = list_concat(tables, relids);
+ tables = list_concat(tables, schemarelids);
+ }
+
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ viaroot = true;
}
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa426..c758e15a7d 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,17 +1759,22 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace,"
+ " LATERAL pg_get_publication_tables(array[ %s ]) gpt"
+ " WHERE c.oid = gpt.relid;",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f00a..371f8abe6c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,10 +11673,10 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
+ provolatile => 's', prorettype => 'oid', proargtypes => 'any',
+ proallargtypes => '{any,oid}', proargmodes => '{i,o}',
proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8315..35f1f4241d 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1439,7 +1439,7 @@ pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
c.relname AS tablename
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables(p.pubname) gpt(relid),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index e7f4a94f19..549252971c 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 0dc0a6d10f..c72a98bed7 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -718,8 +722,11 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e11f3..6e974aa601 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -1040,7 +1040,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -1049,7 +1050,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.18.4
[application/octet-stream] REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (5.4K, ../../OSZPR01MB62789D224DEEBA60924301599ECA9@OSZPR01MB6278.jpnprd01.prod.outlook.com/3-REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From d95019cbaaee049ee3535ef0e6ab455fb2419043 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Thu, 28 Apr 2022 11:28:35 +0800
Subject: [PATCH v4] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, when
subscribing to both publications using one subscription, the data is replicated
twice in inital copy. What we expect is to be copied only once.
To fix this, we exclude the partitioned table whose ancestor belongs to
specified publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 37 +++++++++++++++++++-----
src/test/subscription/t/013_partition.pl | 19 +++++-------
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1719f04517..6ab6f249d0 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1477,7 +1477,8 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
ListCell *lc;
@@ -1486,10 +1487,7 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(list_length(publications) > 0);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
+ initStringInfo(&pub_names);
first = true;
foreach(lc, publications)
{
@@ -1498,14 +1496,37 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
if (first)
first = false;
else
- appendStringInfoString(&cmd, ", ");
+ appendStringInfoString(&pub_names, ", ");
- appendStringInfoString(&cmd, quote_literal_cstr(pubname));
+ appendStringInfoString(&pub_names, quote_literal_cstr(pubname));
}
- appendStringInfoChar(&cmd, ')');
+
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the initial
+ * date in the partitioned table would be replicated twice.
+ */
+ initStringInfo(&cmd);
+ appendStringInfo(&cmd, "SELECT DISTINCT ns.nspname, c.relname\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " JOIN pg_catalog.pg_namespace ns\n"
+ " ON ns.nspname = t.schemaname\n"
+ " JOIN pg_catalog.pg_class c\n"
+ " ON c.relname = t.tablename AND c.relnamespace = ns.oid\n"
+ " WHERE t.pubname IN (%s)\n"
+ " AND (c.relispartition IS FALSE\n"
+ " OR NOT EXISTS\n"
+ " ( SELECT 1 FROM pg_partition_ancestors(c.oid) as pa\n"
+ " WHERE pa.relid IN\n"
+ " (SELECT DISTINCT (t.schemaname || '.' || t.tablename)::regclass::oid\n"
+ " FROM pg_catalog.pg_publication_tables t\n"
+ " WHERE t.pubname IN (%s))\n"
+ " AND pa.relid != c.oid))\n",
+ pub_names.data, pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 39946c735b..c57a8eb7c5 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 67;
+use Test::More tests => 65;
# setup
@@ -471,12 +471,15 @@ $node_subscriber2->safe_psql('postgres',
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
+
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations.
$node_subscriber2->safe_psql('postgres',
@@ -546,10 +549,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -574,10 +573,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-13 14:57 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-13 14:57 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Friday, May 13, 2022 6:42 PM Wang, Wei/王 威 <[email protected]> wrote:
> Attach the patches.(Only changed the patch for HEAD.).
> 1. Optimize the code. Reduce calls to function filter_partitions. [suggestions by
> Amit-san] 2. Improve the alias name in SQL. [suggestions by Amit-san] 3.
> Improve coding alignments. [suggestions by Amit-san] 4. Do some
> optimizations for list Concatenate.
Hi, thank you for updating the patch.
I have one minor comment on fetch_table_list() in HEAD v4.
@@ -1759,17 +1759,22 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
Kindly free the pub_names's data along with the cmd.data.
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-17 13:02 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Amit Kapila @ 2022-05-17 13:02 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, May 13, 2022 at 3:11 PM [email protected]
<[email protected]> wrote:
>
> Attach the patches.(Only changed the patch for HEAD.).
>
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
+ provolatile => 's', prorettype => 'oid', proargtypes => 'any',
+ proallargtypes => '{any,oid}', proargmodes => '{i,o}',
Won't our use case (input one or more publication names) requires the
parameter type to be 'VARIADIC text[]' instead of 'any'? I might be
missing something here so please let me know your reason to change the
type to 'any' from 'text'?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-18 08:37 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-18 08:37 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tues, May 17, 2022 9:03 PM Amit Kapila <[email protected]> wrote:
> On Fri, May 13, 2022 at 3:11 PM [email protected]
> <[email protected]> wrote:
> >
> > Attach the patches.(Only changed the patch for HEAD.).
> >
Thanks for your comments.
> # publications
> -{ oid => '6119', descr => 'get OIDs of tables in a publication',
> +{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
> proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
> - provolatile => 's', prorettype => 'oid', proargtypes => 'text',
> - proallargtypes => '{text,oid}', proargmodes => '{i,o}',
> + provolatile => 's', prorettype => 'oid', proargtypes => 'any',
> + proallargtypes => '{any,oid}', proargmodes => '{i,o}',
>
> Won't our use case (input one or more publication names) requires the
> parameter type to be 'VARIADIC text[]' instead of 'any'? I might be
> missing something here so please let me know your reason to change the
> type to 'any' from 'text'?
Yes, you are right. I improve the approach according to your suggestion.
I didn't notice the field "provariadic" in pg_proc before. And now I found we
could change the type of input from text to variadic text by specifying fields
"provariadic" and specifying 'v' in "proargmodes".
I also make corresponding changes to the processing of the input in function
pg_get_publication_tables.
BTW, in previous patch HEAD_v4, when invoke function GetPublicationByName in
function pg_get_publication_tables, I changed the second input from "false" to
"true". I changed this because when we invoke function
pg_get_publication_tables in the query in function fetch_table_list, if the
publication does not exist, it will error.
But this change will affect the compatibility of function
pg_get_publication_tables. So I revert this change in HEAD_v5, and filter the
publications that do not exist by the query in function fetch_table_list.
Attach the patches.(Only changed the patch for HEAD.)
1. Improve the approach to modify the input type of the function
pg_get_publication_tables. [suggestions by Amit-san]
2. Free allocated memory in function fetch_table_list. [suggestions by Osumi-san]
3. Improve the approach of the handling of non-existing publications.
BTW, I rename the patch for REL14
from
"REL14_v4-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch"
to
"REL14_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch".
Just for the version doesn't mess up between two branches and for cfbot.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (13.1K, ../../OS3PR01MB6275B26B6BDF23651B8CE86D9ED19@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From e5583305d2e13abb229e26d163d0e38524609ff6 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Fri, 13 May 2022 09:27:41 +0800
Subject: [PATCH v5] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
To fix this, we extended the API of the function pg_get_publication_tables.
(Change the parameter type from text to any).
So, the function pg_get_publication_tables could also receive the array of
publications. And then, if we specify option viaroot, we could exclude the
partitioned table whose ancestor belongs to this publications array when
getting the table list.
---
src/backend/catalog/pg_publication.c | 88 +++++++++++++---------
src/backend/commands/subscriptioncmds.c | 20 +++--
src/include/catalog/pg_proc.dat | 11 +--
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 ++---
src/test/subscription/t/028_row_filter.pl | 11 ++-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 92 insertions(+), 64 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb279..367dc0ba3b 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,20 +1077,24 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns Oids of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems;
+ Publication *publication;
+ int i;
+ bool viaroot = false;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1098,44 +1102,60 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* get Oids of tables from each publication */
+ for (i = 0; i < nelems; i++)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
- {
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
*/
+ if (publication->alltables)
+ tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ tables = list_concat(tables, relids);
+ tables = list_concat(tables, schemarelids);
+ }
+
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ viaroot = true;
}
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa426..e3e64ca848 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,19 +1759,27 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace"
+ " WHERE c.oid IN"
+ " (SELECT pg_get_publication_tables(VARIADIC array_agg(pubname::text))"
+ " FROM pg_publication"
+ " WHERE pubname IN ( %s ));",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f00a..9d59dcafbb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,12 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'oid', proargtypes => '_text', proallargtypes => '{_text,oid}',
+ proargmodes => '{v,o}', proargnames => '{pubname,relid}',
+ prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8315..f754ffffc4 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1439,7 +1439,7 @@ pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
c.relname AS tablename
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index e7f4a94f19..549252971c 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 0dc0a6d10f..c72a98bed7 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -718,8 +722,11 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e11f3..6e974aa601 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -1040,7 +1040,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -1049,7 +1050,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.18.4
[application/octet-stream] REL14_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch (5.4K, ../../OS3PR01MB6275B26B6BDF23651B8CE86D9ED19@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL14_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-18 08:39 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-05-18 08:39 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, May 13, 2022 10:57 PM Osumi, Takamichi/大墨 昂道 <[email protected]> wrote:
> On Friday, May 13, 2022 6:42 PM Wang, Wei/王 威 <[email protected]>
> wrote:
> > Attach the patches.(Only changed the patch for HEAD.).
> > 1. Optimize the code. Reduce calls to function filter_partitions.
> > [suggestions by Amit-san] 2. Improve the alias name in SQL. [suggestions by
> Amit-san] 3.
> > Improve coding alignments. [suggestions by Amit-san] 4. Do some
> > optimizations for list Concatenate.
> Hi, thank you for updating the patch.
>
>
> I have one minor comment on fetch_table_list() in HEAD v4.
Thanks for your comments.
> @@ -1759,17 +1759,22 @@ static List *
> fetch_table_list(WalReceiverConn *wrconn, List *publications) {
> WalRcvExecResult *res;
> - StringInfoData cmd;
> + StringInfoData cmd,
> + pub_names;
> TupleTableSlot *slot;
> Oid tableRow[2] = {TEXTOID, TEXTOID};
> List *tablelist = NIL;
>
> + initStringInfo(&pub_names);
> + get_publications_str(publications, &pub_names, true);
> +
>
> Kindly free the pub_names's data along with the cmd.data.
Improve it according to your suggestion. Free 'pub_names.data'.
I also made some other changes.
Kindly have a look at new patch shared in [1].
[1] https://www.postgresql.org/message-id/OS3PR01MB6275B26B6BDF23651B8CE86D9ED19%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-05-18 08:51 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-05-18 08:51 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wed, May 18, 2022 4:38 PM I wrote:
> Attach the patches.(Only changed the patch for HEAD.)
Sorry, I forgot to update commit message.
Attach the new patch.
1. Only update the commit message for HEAD_v5.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (13.1K, ../../OS3PR01MB6275937EAEE44CC137711F489ED19@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 9d2b5743e7c34e20ac67505d1da12c7e021d5740 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 18 May 2022 16:54:01 +0800
Subject: [PATCH v5] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
To fix this, we extended the API of the function pg_get_publication_tables.
Now, the function pg_get_publication_tables could receive the publication list.
And then, if we specify option viaroot, we could exclude the partitioned table
whose ancestor belongs to the publication list when getting the table list.
---
src/backend/catalog/pg_publication.c | 88 +++++++++++++---------
src/backend/commands/subscriptioncmds.c | 20 +++--
src/include/catalog/pg_proc.dat | 11 +--
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 ++---
src/test/subscription/t/028_row_filter.pl | 11 ++-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 92 insertions(+), 64 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb279..367dc0ba3b 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,20 +1077,24 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns Oids of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems;
+ Publication *publication;
+ int i;
+ bool viaroot = false;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1098,44 +1102,60 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* get Oids of tables from each publication */
+ for (i = 0; i < nelems; i++)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
- {
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
*/
+ if (publication->alltables)
+ tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ tables = list_concat(tables, relids);
+ tables = list_concat(tables, schemarelids);
+ }
+
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ viaroot = true;
}
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa426..e3e64ca848 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1759,19 +1759,27 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
List *tablelist = NIL;
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname"
+ " FROM pg_class c JOIN pg_namespace n"
+ " ON n.oid = c.relnamespace"
+ " WHERE c.oid IN"
+ " (SELECT pg_get_publication_tables(VARIADIC array_agg(pubname::text))"
+ " FROM pg_publication"
+ " WHERE pubname IN ( %s ));",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f00a..9d59dcafbb 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,12 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+{ oid => '6119', descr => 'get OIDs of tables in one or more publications',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'oid', proargtypes => '_text', proallargtypes => '{_text,oid}',
+ proargmodes => '{v,o}', proargnames => '{pubname,relid}',
+ prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8315..f754ffffc4 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1439,7 +1439,7 @@ pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
c.relname AS tablename
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index e7f4a94f19..549252971c 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index 0dc0a6d10f..c72a98bed7 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -718,8 +722,11 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e11f3..6e974aa601 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -1040,7 +1040,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -1049,7 +1050,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.18.4
[application/octet-stream] REL14_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch (5.4K, ../../OS3PR01MB6275937EAEE44CC137711F489ED19@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL14_v5-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-07-01 09:46 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-07-01 09:46 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wed, May 18, 2022 4:51 PM I wrote:
> Attach the new patch.
Since there are some new commits in HEAD (0ff20288, fd0b9dc and 52b5c53) that
improve the functions pg_get_publication_tables and fetch_table_list, we cannot
apply the patch cleanly. Therefore, I rebased the patch based on the changes in
HEAD.
I also rebased the patch for REL14 because the commit 52d5ea9 in branch
REL_14_STABLE. BTW, I made a slight adjustments in the function
fetch_table_list to the SQL used to get the publisher-side table information
for version 14.
Since we have REL_15_STABLE now, I also attach the patch for version 15.
Attach the patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch (17.8K, ../../OS3PR01MB6275AFA91925615A4AA782D09EBD9@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH.patch)
download | inline diff:
From 6293d73363082ad3517074d8bf4c2459f649e9fb Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v6] Fix data replicated twice when specifying
PUBLISH_VIA_PARTITION_ROOT option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
To fix this, we extend the API of the function pg_get_publication_tables.
Now, the function pg_get_publication_tables could receive the publication list.
And then, if we specify option viaroot, we could exclude the partitioned table
whose ancestor belongs to the publication list when getting the table
informations.
---
src/backend/catalog/pg_publication.c | 145 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 65 +++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 +--
src/test/subscription/t/028_row_filter.pl | 11 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 186 insertions(+), 73 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index c365de3a72..f095bf40a8 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,22 +1077,36 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Returns information of tables in one or more publications.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL,
+ *results = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
+ typedef struct
+ {
+ Oid pubid;
+ List *table_list;
+ } pub_info;
+
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ Publication *publication;
+ bool viaroot = false;
+ List *pub_infos = NIL;
+ ListCell *lc1,
+ *lc2;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1100,43 +1114,93 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* get Oids of tables from each publication */
+ for (i = 0; i < nelems; i++)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
- {
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ List *current_tables = NIL;
+ pub_info *pubinfo = (pub_info *) malloc(sizeof(pub_info));
+
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
*/
+ if (publication->alltables)
+ current_tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_tables = list_concat(relids, schemarelids);
+ }
+
+ /* Record the publication and corresponding table list. */
+ pubinfo->pubid = publication->oid;
+ pubinfo->table_list = list_copy(current_tables);
+ pub_infos = lappend(pub_infos, pubinfo);
+
+ tables = list_concat(tables, current_tables);
+
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc1, tables)
+ {
+ Oid relid = lfirst_oid(lc1);
+
+ foreach(lc2, pub_infos)
+ {
+ pub_info *pubinfo = (pub_info *) lfirst(lc2);
+
+ if (list_member_oid(pubinfo->table_list, relid))
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = relid;
+ result[1] = pubinfo->pubid;
+
+ results = lappend(results, result);
+ }
+ }
}
+ list_free(pub_infos);
/* Construct a tuple descriptor for the result rows. */
tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATION_TABLES_ELEM);
@@ -1148,20 +1212,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) results;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ results = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(results))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Oid *table_info = (Oid *) list_nth(results, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
+ Publication *publication;
Datum values[NUM_PUBLICATION_TABLES_ELEM];
bool nulls[NUM_PUBLICATION_TABLES_ELEM];
@@ -1171,7 +1238,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
MemSet(nulls, 0, sizeof(nulls));
MemSet(values, 0, sizeof(values));
- publication = GetPublicationByName(pubname, false);
+ publication = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 83e6eae855..04aa83da95 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1764,25 +1764,70 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
/* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ if (check_columnlist && server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname,\n"
+ " ( CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class c\n"
+ " JOIN pg_namespace n ON n.oid = c.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as gpt\n"
+ " ON gpt.relid = c.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the
+ * initial date in the partitioned table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as pa\n"
+ " WHERE pa.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND pa.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a77b293723..6622401320 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of tables in one or more publications',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fc3cde3226..2a9a9c9da4 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1448,7 +1448,7 @@ pg_publication_tables| SELECT p.pubname,
JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 0dfbbabc3b..ae6e2057c8 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index b1fb2d7cae..d7dcecf960 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -723,8 +727,11 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 9fa6e0b35f..3d07b456bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -973,7 +973,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -982,7 +983,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.23.0.windows.1
[application/octet-stream] REL15_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch (16.7K, ../../OS3PR01MB6275AFA91925615A4AA782D09EBD9@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch)
download
[application/octet-stream] REL14_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch (5.4K, ../../OS3PR01MB6275AFA91925615A4AA782D09EBD9@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v6-0001-Fix-data-replicated-twice-when-specifying-PUBLISH_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-07-14 04:45 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-07-14 04:45 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are some review comments for the v6 patch (HEAD only):
============
HEAD_v6-0001
============
1. Commit message
If there are two publications that publish the parent table and the child table
separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT, subscribing
to both publications from one subscription causes initial copy twice. What we
expect is to be copied only once.
~
I don’t think the parameter even works in uppercase, so maybe better to say:
PUBLISH_VIA_PARTITION_ROOT -> 'publish_via_partition_root'
~~~
2.
What we expect is to be copied only once.
SUGGESTION
It should only be copied once.
~~~
3.
To fix this, we extend the API of the function pg_get_publication_tables.
Now, the function pg_get_publication_tables could receive the publication list.
And then, if we specify option viaroot, we could exclude the partitioned table
whose ancestor belongs to the publication list when getting the table
informations.
~
Don't you mean "partition table" instead of "partitioned table"?
SUGGESTION (also reworded)
To fix this, the API function pg_get_publication_tables has been
extended to take a publication list. Now, when getting the table
information, if the publish_via_partition_root is true, the function
can exclude a partition table whose ancestor is also published by the
same publication list.
======
4. src/backend/catalog/pg_publication.c - pg_get_publication_tables
- publication = GetPublicationByName(pubname, false);
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
Maybe should have some comment to describe that this function
parameter is now an array of publications names.
~~~
5.
+ /* get Oids of tables from each publication */
Uppercase comment
~~~
6.
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ Publication *publication;
+ bool viaroot = false;
+ List *pub_infos = NIL;
+ ListCell *lc1,
+ *lc2;
The 'publication' should be declared only in the loop that uses it.
It's also not good that this is shadowing the same variable name in a
later declaration.
~~~
7.
+ * Publications support partitioned tables, although all changes
+ * are replicated using leaf partition identity and schema, so we
+ * only need those.
*/
+ if (publication->alltables)
+ current_tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_tables = list_concat(relids, schemarelids);
+ }
Somehow I was confused by this comment because it says you only need
the LEAF tables but then the subsequent code is getting ROOT relations
anyway... Can you clarify the comment some more?
~~~
8.
+ bool viaroot = false;
I think that should have a comment something like:
/* At least one publication is using publish_via_partition_root */
~~~
9.
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc1, tables)
+ {
+ Oid relid = lfirst_oid(lc1);
+
+ foreach(lc2, pub_infos)
+ {
+ pub_info *pubinfo = (pub_info *) lfirst(lc2);
+
+ if (list_member_oid(pubinfo->table_list, relid))
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = relid;
+ result[1] = pubinfo->pubid;
+
+ results = lappend(results, result);
+ }
+ }
}
I felt a bit uneasy about the double-looping here. I wonder if these
'results' could have been accumulated within the existing loop over
all publications. Then the results would need to be filtered to remove
the ones associated with removed partitions. Otherwise with 10000
tables and also many publications this (current) double-looping seems
like it might be quite expensive.
======
10. src/backend/commands/subscriptioncmds.c - fetch_table_list
+ if (check_columnlist && server_version >= 160000)
This condition does not make much sense to me. Isn’t it effectively
same as saying
if (server_version >= 150000 && server_version >= 160000)
???
~~~
11.
+ /*
+ * Get the list of tables from publisher, the partitioned table whose
+ * ancestor is also in this list should be ignored, otherwise the
+ * initial date in the partitioned table would be replicated twice.
+ */
11.a
Isn't this comment all backwards? I think you mean to say "partition"
or "partition table" (not partitioned table) because partitions have
ancestors but partition-ED tables don't.
11.b
"initial date" -> "initial data"
======
12. src/test/subscription/t/013_partition.pl
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partition table (tab4) here.
+# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
+# pub_lower_level above), all data should be replicated to the partition table.
+# So we do not need to create table for the partitioned table.
12.a
AFAIK "tab4" is the *partitioned* table, not a partition. I think this
comment has all the "partitioned" and "partition" back-to-front.
12.b
Also please say “publish_via_partition_root" instead of
PUBLISH_VIA_PARTITION_ROOT
======
13. src/test/subscription/t/028_row_filter.pl
@@ -723,8 +727,11 @@ is($result, qq(t|1), 'check replicated rows to
tab_rowfilter_toast');
# - INSERT (16) YES, 16 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
There is a comment above that code like:
# tab_rowfilter_viaroot_part filter is: (a > 15)
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
I think should modify that comment to explain the new data this patch
inserts - e.g. NO for 13 and YES for 17...
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-07-21 10:07 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-07-21 10:07 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Thur, Jul 14, 2022 at 12:46 PM Peter Smith <[email protected]> wrote:
> Here are some review comments for the v6 patch (HEAD only):
Thanks for your comments.
> 1. Commit message
>
> If there are two publications that publish the parent table and the child table
> separately, and both specify the option PUBLISH_VIA_PARTITION_ROOT,
> subscribing
> to both publications from one subscription causes initial copy twice. What we
> expect is to be copied only once.
>
> ~
>
> I don’t think the parameter even works in uppercase, so maybe better to say:
> PUBLISH_VIA_PARTITION_ROOT -> 'publish_via_partition_root'
It seems that there are more places to use lowercase than uppercase, so
improved it as suggested.
> 2.
>
> What we expect is to be copied only once.
>
> SUGGESTION
> It should only be copied once.
>
> ~~~
>
> 3.
>
> To fix this, we extend the API of the function pg_get_publication_tables.
> Now, the function pg_get_publication_tables could receive the publication list.
> And then, if we specify option viaroot, we could exclude the partitioned table
> whose ancestor belongs to the publication list when getting the table
> informations.
>
> ~
>
> Don't you mean "partition table" instead of "partitioned table"?
>
> SUGGESTION (also reworded)
> To fix this, the API function pg_get_publication_tables has been
> extended to take a publication list. Now, when getting the table
> information, if the publish_via_partition_root is true, the function
> can exclude a partition table whose ancestor is also published by the
> same publication list.
Improved and fixed as suggested.
> 4. src/backend/catalog/pg_publication.c - pg_get_publication_tables
>
> - publication = GetPublicationByName(pubname, false);
> + arr = PG_GETARG_ARRAYTYPE_P(0);
> + deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
> + &elems, NULL, &nelems);
>
> Maybe should have some comment to describe that this function
> parameter is now an array of publications names.
Add the following comment: `/* Deconstruct the parameter into elements. */`.
Also improved the comment above the function pg_get_publication_tables:
`Returns information of tables in one or more publications.`
-->
`Returns information of the tables in the given publication array.`
> 5.
>
> + /* get Oids of tables from each publication */
>
> Uppercase comment
Improved as suggested.
> 6.
>
> + ArrayType *arr;
> + Datum *elems;
> + int nelems,
> + i;
> + Publication *publication;
> + bool viaroot = false;
> + List *pub_infos = NIL;
> + ListCell *lc1,
> + *lc2;
>
> The 'publication' should be declared only in the loop that uses it.
> It's also not good that this is shadowing the same variable name in a
> later declaration.
Reverted changes to variable "publication" declarations.
> 7.
>
> + * Publications support partitioned tables, although all changes
> + * are replicated using leaf partition identity and schema, so we
> + * only need those.
> */
> + if (publication->alltables)
> + current_tables = GetAllTablesPublicationRelations(publication->pubviaroot);
> + else
> + {
> + List *relids,
> + *schemarelids;
> +
> + relids = GetPublicationRelations(publication->oid,
> + publication->pubviaroot ?
> + PUBLICATION_PART_ROOT :
> + PUBLICATION_PART_LEAF);
> + schemarelids = GetAllSchemaPublicationRelations(publication->oid,
> + publication->pubviaroot ?
> + PUBLICATION_PART_ROOT :
> + PUBLICATION_PART_LEAF);
> + current_tables = list_concat(relids, schemarelids);
> + }
>
> Somehow I was confused by this comment because it says you only need
> the LEAF tables but then the subsequent code is getting ROOT relations
> anyway... Can you clarify the comment some more?
I think this is a slight mistake when publication parameter
"publish_via_partition_root" was introduced before.
I improved the comment to the following:
```
Publications support partitioned tables. If
publish_via_partition_root is false, all changes are replicated
using leaf partition identity and schema, so we only need those.
Otherwise, If publish_via_partition_root is true, get the
partitioned table itself.
```
> 8.
>
> + bool viaroot = false;
>
> I think that should have a comment something like:
> /* At least one publication is using publish_via_partition_root */
Improved as suggested.
> 9.
>
> + /*
> + * Record the published table and the corresponding publication so
> + * that we can get row filters and column list later.
> + */
> + foreach(lc1, tables)
> + {
> + Oid relid = lfirst_oid(lc1);
> +
> + foreach(lc2, pub_infos)
> + {
> + pub_info *pubinfo = (pub_info *) lfirst(lc2);
> +
> + if (list_member_oid(pubinfo->table_list, relid))
> + {
> + Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
> +
> + result[0] = relid;
> + result[1] = pubinfo->pubid;
> +
> + results = lappend(results, result);
> + }
> + }
> }
>
> I felt a bit uneasy about the double-looping here. I wonder if these
> 'results' could have been accumulated within the existing loop over
> all publications. Then the results would need to be filtered to remove
> the ones associated with removed partitions. Otherwise with 10000
> tables and also many publications this (current) double-looping seems
> like it might be quite expensive.
Improved as suggested.
> 10. src/backend/commands/subscriptioncmds.c - fetch_table_list
>
> + if (check_columnlist && server_version >= 160000)
>
> This condition does not make much sense to me. Isn’t it effectively
> same as saying
> if (server_version >= 150000 && server_version >= 160000)
>
> ???
Fixed as suggested.
> 11.
>
> + /*
> + * Get the list of tables from publisher, the partitioned table whose
> + * ancestor is also in this list should be ignored, otherwise the
> + * initial date in the partitioned table would be replicated twice.
> + */
>
> 11.a
> Isn't this comment all backwards? I think you mean to say "partition"
> or "partition table" (not partitioned table) because partitions have
> ancestors but partition-ED tables don't.
>
>
> 11.b
> "initial date" -> "initial data"
Fixed as suggested.
> 12. src/test/subscription/t/013_partition.pl
>
> -# Note: We create two separate tables, not a partitioned one, so that we can
> -# easily identity through which relation were the changes replicated.
> +# Note: We only create one table for the partition table (tab4) here.
> +# Because we specify option PUBLISH_VIA_PARTITION_ROOT (see pub_all and
> +# pub_lower_level above), all data should be replicated to the partition table.
> +# So we do not need to create table for the partitioned table.
>
> 12.a
> AFAIK "tab4" is the *partitioned* table, not a partition. I think this
> comment has all the "partitioned" and "partition" back-to-front.
>
> 12.b
> Also please say “publish_via_partition_root" instead of
> PUBLISH_VIA_PARTITION_ROOT
Fixed as suggested.
> 13. src/test/subscription/t/028_row_filter.pl
>
> @@ -723,8 +727,11 @@ is($result, qq(t|1), 'check replicated rows to
> tab_rowfilter_toast');
> # - INSERT (16) YES, 16 > 15
> $result =
> $node_subscriber->safe_psql('postgres',
> - "SELECT a FROM tab_rowfilter_viaroot_part");
> -is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
> + "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
> +is($result, qq(16
> +17),
> + 'check replicated rows to tab_rowfilter_viaroot_part'
> +);
>
> There is a comment above that code like:
> # tab_rowfilter_viaroot_part filter is: (a > 15)
> # - INSERT (14) NO, 14 < 15
> # - INSERT (15) NO, 15 = 15
> # - INSERT (16) YES, 16 > 15
>
> I think should modify that comment to explain the new data this patch
> inserts - e.g. NO for 13 and YES for 17...
Improved as suggested.
I also improved the patches for back-branch according to some of Peter's
comments and added the back-branch patch for REL_13.
In addition, in the patch (REL15_v6) I attached for REL15 in [1], I forgot to
remove the modification to the function pg_get_publication_tables. I removed
related modifications now (REL15_v7).
Attach the new patches.
[1] - https://www.postgresql.org/message-id/OS3PR01MB6275AFA91925615A4AA782D09EBD9%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v7-0001-Fix-data-replicated-twice-when-specifying-publish.patch (17.7K, ../../OS3PR01MB6275F576802D9FD34C2836849E919@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v7-0001-Fix-data-replicated-twice-when-specifying-publish.patch)
download | inline diff:
From 634247cff61908b6c63c29b242bdaef5ce28767b Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v7] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications that publish the parent table and the child table
separately, and both specify the option publish_via_partition_root, subscribing
to both publications from one subscription causes initial copy twice. It should
only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 133 ++++++++++++++-------
src/backend/commands/subscriptioncmds.c | 65 ++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 19 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 175 insertions(+), 74 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index ade3bf3aca..44e95d7c1f 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,22 +1077,28 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Returns information of the tables in the given publication array.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ Publication *publication;
+ List *tables = NIL,
+ *results = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1100,42 +1106,87 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
+ /* Deconstruct the parameter into elements. */
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
- {
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ List *current_tables = NIL;
+
+ publication = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, If publish_via_partition_root is true, get the
+ * partitioned table itself.
*/
+ if (publication->alltables)
+ current_tables = GetAllTablesPublicationRelations(publication->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(publication->oid,
+ publication->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, current_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = publication->oid;
+ results = lappend(results, result);
+ }
+
+ tables = list_concat(tables, current_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /* Filter by final published table. */
+ foreach(lc, results)
+ {
+ Oid *table_info = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info[0]))
+ results = foreach_delete_current(results, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1148,20 +1199,22 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) results;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ results = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(results))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Oid *table_info = (Oid *) list_nth(results, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1169,7 +1222,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ publication = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bd0cc0848d..cca862adb6 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1794,25 +1794,70 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
/* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname,\n"
+ " ( CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class c\n"
+ " JOIN pg_namespace n ON n.oid = c.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as gpt\n"
+ " ON gpt.relid = c.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list should be ignored, otherwise the
+ * initial data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as pa\n"
+ " WHERE pa.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND pa.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 2e41f4d9e8..8275ba9d4e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of tables in one or more publications',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7ec3d2688f..54f5ce5da9 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1449,7 +1449,7 @@ pg_publication_tables| SELECT p.pubname,
JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 0dfbbabc3b..008cea6181 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -477,12 +477,13 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table for the partitioned table (tab4) here. Because
+# we specify option "publish_via_partition_root" (see pub_all and
+# pub_lower_level above), all data should be replicated to the partitioned
+# table. So we do not need to create table for the partition table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -556,11 +557,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -584,11 +580,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index b1fb2d7cae..043b5031fc 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -393,6 +393,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -718,13 +722,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 9fa6e0b35f..3d07b456bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -973,7 +973,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -982,7 +983,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
wait_for_subscription_sync($node_subscriber);
--
2.23.0.windows.1
[application/octet-stream] REL15_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch (8.5K, ../../OS3PR01MB6275F576802D9FD34C2836849E919@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL14_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.3K, ../../OS3PR01MB6275F576802D9FD34C2836849E919@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL13_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.3K, ../../OS3PR01MB6275F576802D9FD34C2836849E919@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v7-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-07-28 09:17 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: Peter Smith @ 2022-07-28 09:17 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are some review comments for the HEAD_v7-0001 patch:
======
1. <General>
I have a fundamental question about this patch.
IIUC the purpose of this patch is to ensure that (when
publish_via_root = true) the copy of the partition data will happen
only once (e.g. from one parent table on one of the publishers). But I
think there is no guarantee that those 2 publishers even had the same
data, right? Therefore it seems to me you could get different results
if the data were copied from pub1 or from pub2. (I have not tried it -
this is just my suspicion).
Am I correct or mistaken? If correct, then it means there is a big
(but subtle) difference related to the ordering of processing ... A)
is this explicitly documented so the user knows what data to expect?
B) is the effect of different ordering tested anywhere? Actually, I
have no idea what exactly determines the order – is it the original
CREATE SUBSCRIPTION publication list order? Is it the logic of the
pg_get_publication_tables function? Is it the SQL in function
fetch_table_list? Or is it not deterministic at all? Please confirm
it.
======
2. Commit message.
2a.
If there are two publications that publish the parent table and the child table
separately, and both specify the option publish_via_partition_root, subscribing
to both publications from one subscription causes initial copy twice.
SUGGESTION
If there are two publications that publish the parent table and the child table
respectively, but both specify publish_via_partition_root = true, subscribing
to both publications from one subscription causes initial copy twice.
2b. <General>
Actually, isn't it more subtle than what that comment is describing?
Maybe nobody is explicitly publishing a parent table at all. Maybe
pub1 publishes partition1 and pub2 publishes partition2, but both
publications are using publish_via_partition_root = true. Is this
scenario even tested? Does the logic of pg_get_publication_tables
cover this scenario?
======
3. src/backend/catalog/pg_publication.c - pg_get_publication_tables
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ Publication *publication;
Something seems strange about having a common Publication declaration.
Firstly it is used to represent every publication element in the array
loop. Later, it is overwritten to represent a single publication.
I think it might be easier if you declare these variables separately:
E.g.1
Publication *pub_elem; -- for the array element processing declared
within the for loop
E.g.2
Publication *pub; -- declared within if (funcctx->call_cntr <
list_length(results))
~~~
4.
+ /* Filter by final published table. */
+ foreach(lc, results)
+ {
+ Oid *table_info = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info[0]))
+ results = foreach_delete_current(results, lc);
}
The comment did not convey enough meaning. Can you make it more
descriptive to explain why/what the logic is doing here?
======
5. src/backend/commands/subscriptioncmds.c - fetch_table_list
/* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname,\n"
That comment is exactly the same as it was before the patch. But it
doesn't seem quite appropriate anymore for this new condition and this
new query.
~~~
6.
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list should be ignored, otherwise the
+ * initial data in the partition table would be replicated twice.
+ */
Why say "should be ignored" -- don’t you mean "will be" or "must be" or "is".
~~~
7.
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
/* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname,\n"
+ " ( CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class c\n"
+ " JOIN pg_namespace n ON n.oid = c.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC
array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as gpt\n"
+ " ON gpt.relid = c.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list should be ignored, otherwise the
+ * initial data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(gpt.attrs,
1) = c.relnatts)\n"
+ " THEN NULL ELSE gpt.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM
pg_partition_ancestors(pub_tabs.oid) as pa\n"
+ " WHERE pa.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND pa.relid != pub_tabs.oid))\n");
+ }
Please use a consistent case for all the SQL aliases. E.g "gpt" versus
"GPT", "c" versus "C", etc.
======
8. src/test/subscription/t/013_partition.pl
+# Note: We only create one table for the partitioned table (tab4) here. Because
+# we specify option "publish_via_partition_root" (see pub_all and
+# pub_lower_level above), all data should be replicated to the partitioned
+# table. So we do not need to create table for the partition table.
"replicated to the partitioned table" ??
The entire comment seems a bit misleading because how can we call the
subscriber table a "partitioned" table when it has no partitions?!
SUGGESTION (maybe?)
Note: We only create one table (tab4) here. We specified
publish_via_partition_root = true (see pub_all and pub_lower_level
above), so all data will be replicated to that table.
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-04 11:26 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
1 sibling, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-08-04 11:26 UTC (permalink / raw)
To: Peter Smith <[email protected]>; [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Thursday, July 28, 2022 5:17 PM Peter Smith <[email protected]> wrote:
> Here are some review comments for the HEAD_v7-0001 patch:
>
> ======
>
> 1. <General>
>
> I have a fundamental question about this patch.
>
> IIUC the purpose of this patch is to ensure that (when
> publish_via_root = true) the copy of the partition data will happen
> only once (e.g. from one parent table on one of the publishers). But I
> think there is no guarantee that those 2 publishers even had the same
> data, right? Therefore it seems to me you could get different results
> if the data were copied from pub1 or from pub2. (I have not tried it -
> this is just my suspicion).
>
> Am I correct or mistaken?
Since the subscribed publishers are combined with OR and are from the same database.
And we are trying to copy the data from the top most parent table, so I think the results
should be as expected.
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-05 09:06 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
1 sibling, 2 replies; 99+ messages in thread
From: [email protected] @ 2022-08-05 09:06 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Thur, Jul 28, 2022 at 17:17 PM Peter Smith <[email protected]> wrote:
> Here are some review comments for the HEAD_v7-0001 patch:
Thanks for your comments.
> 2. Commit message.
>
> 2a.
>
> If there are two publications that publish the parent table and the child table
> separately, and both specify the option publish_via_partition_root, subscribing
> to both publications from one subscription causes initial copy twice.
>
> SUGGESTION
> If there are two publications that publish the parent table and the child table
> respectively, but both specify publish_via_partition_root = true, subscribing
> to both publications from one subscription causes initial copy twice.
>
> 2b. <General>
>
> Actually, isn't it more subtle than what that comment is describing?
> Maybe nobody is explicitly publishing a parent table at all. Maybe
> pub1 publishes partition1 and pub2 publishes partition2, but both
> publications are using publish_via_partition_root = true. Is this
> scenario even tested? Does the logic of pg_get_publication_tables
> cover this scenario?
=>2a.
Okay, changed it as suggested.
=>2b.
This is not the case we are trying to fix. The problematic scenario is when the
a parent table is published via root partitioned table and in this case we need
to ignore other partitions. And I try to improve the commit message to make it
clear.
> 4.
>
> + /* Filter by final published table. */
> + foreach(lc, results)
> + {
> + Oid *table_info = (Oid *) lfirst(lc);
> +
> + if (!list_member_oid(tables, table_info[0]))
> + results = foreach_delete_current(results, lc);
> }
>
> The comment did not convey enough meaning. Can you make it more
> descriptive to explain why/what the logic is doing here?
I think the comments above `tables = filter_partitions(tables);` explain this.
> 5. src/backend/commands/subscriptioncmds.c - fetch_table_list
>
> /* Get column lists for each relation if the publisher supports it */
> - if (check_columnlist)
> - appendStringInfoString(&cmd, ", t.attnames\n");
> + if (server_version >= 160000)
> + appendStringInfo(&cmd, "SELECT DISTINCT n.nspname, c.relname,\n"
>
> That comment is exactly the same as it was before the patch. But it
> doesn't seem quite appropriate anymore for this new condition and this
> new query.
Improved the comments as following:
```
Get information of the tables belonging to the specified publications
```
The rest of the comments are improved as suggested.
I also rebased the patch based on the commit (0c20dd3) on HEAD, and made some
changes to the back-branch patches based on some of Peter's comments.
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v8-0001-Fix-data-replicated-twice-when-specifying-publish.patch (18.0K, ../../OS3PR01MB627506056B78D533FBB331DC9E9E9@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v8-0001-Fix-data-replicated-twice-when-specifying-publish.patch)
download | inline diff:
From c7d8a9a5b124ae808263c3a5773c3925196deb7a Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v8] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications, one of them publish a parent table with
(publish_via_partition_root = true) and another publish child table,
subscribing to both publications from one subscription results in two initial
replications. It should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 138 ++++++++++++++-------
src/backend/commands/subscriptioncmds.c | 67 ++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 178 insertions(+), 77 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 6af3570005..93e8e03bab 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,27 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Returns information of the tables in the given publication array.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ List *tables = NIL,
+ *results = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1054,88 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
+ /* Deconstruct the parameter into elements. */
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
- /*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
- */
- if (publication->alltables)
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
- {
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *current_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, If publish_via_partition_root is true, get the
+ * partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ current_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ current_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, current_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = pub_elem->oid;
+ results = lappend(results, result);
+ }
+
+ tables = list_concat(tables, current_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /* Filter by final published table */
+ foreach(lc, results)
+ {
+ Oid *table_info = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info[0]))
+ results = foreach_delete_current(results, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1148,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) results;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ results = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(results))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ Oid *table_info = (Oid *) list_nth(results, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1118,13 +1172,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f73dfb6067..163b79291a 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1794,25 +1794,70 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ /* Get information of the tables belonging to the specified publications */
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index be47583122..33f40e81d6 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11681,11 +11681,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of tables in one or more publications',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7ec3d2688f..54f5ce5da9 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1449,7 +1449,7 @@ pg_publication_tables| SELECT p.pubname,
JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index f5f8a67092..e0d388d37f 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index b6644556cf..9f88b9ef52 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.23.0.windows.1
[application/octet-stream] REL15_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch (8.4K, ../../OS3PR01MB627506056B78D533FBB331DC9E9E9@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL14_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.2K, ../../OS3PR01MB627506056B78D533FBB331DC9E9E9@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL13_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.2K, ../../OS3PR01MB627506056B78D533FBB331DC9E9E9@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v8-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-09 07:14 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-08-09 07:14 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are some review comment for the HEAD_v8 patch:
======
1. Commit message
If there are two publications, one of them publish a parent table with
(publish_via_partition_root = true) and another publish child table,
subscribing to both publications from one subscription results in two initial
replications. It should only be copied once.
SUGGESTION (Note**)
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing
one of the parent's child tables - then subscribing to both
publications from one subscription results in the same initial child
data being copied twice. It should only be copied once.
Note** - I've only reworded the original commit message slightly but
otherwise left it saying the same thing. But I still have doubts if
this message actually covers all the cases the patch is trying to
address. e.g. The comment (see below) in the 'fetch_table_list'
function seemed to cover more cases than what this commit message is
describing.
/*
* Get the list of tables from publisher, the partition table whose
* ancestor is also in this list will be ignored, otherwise the initial
* data in the partition table would be replicated twice.
*/
======
2. src/backend/catalog/pg_publication.c - pg_get_publication_tables
2a.
/*
- * Returns information of tables in a publication.
+ * Returns information of the tables in the given publication array.
*/
What does "information of the tables" actually mean? Is it just the
names of the tables; is it more than that? IMO a longer, more
explanatory comment will be better here instead of a brief ambiguous
one.
2b.
+ *results = NIL;
This variable name 'results' is too generic, so it is not helpful when
trying to understand the subsequent code logic. Please give this a
meaningful name/description.
2c.
/* Deconstruct the parameter into elements. */
SUGGESTION
Deconstruct the parameter into elements where each element is a
publication name.
2d.
+ List *current_tables = NIL;
I think this is the tables only on the current pub_elem, so I thought
'pub_elem_tables' might make it easier to understand this list's
meaning.
2e.
+ /* Now sort and de-duplicate the result list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
IMO this comment is confusing because there is another list that is
called the 'results' list, but that is not the same list you are
processing here. Also, it does not really add anything helpful to just
have comments saying almost the same as the function names
(sort/de-duplicate), so please give longer comments to say the reason
*why* the logic does this rather than just describing the steps.
2f.
+ /* Filter by final published table */
+ foreach(lc, results)
Please make this comment more descriptive to explain why/what the
logic is doing.
======
3. src/backend/commands/subscriptioncmds.c - fetch_table_list
3a.
+ bool check_columnlist = (server_version >= 150000);
Given the assignment, maybe 'columnlist_supported' is a better name?
3b.
+ /* Get information of the tables belonging to the specified publications */
For "Get information of..." can you elaborate *what* table
information this is getting and why?
3c.
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC
array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
AFAICT the main reason for this check was to decide if you can use the
new version of 'pg_get_publication_tables' that supports the VARIADIC
array of pub names or not. If that is correct, then maybe the comment
should describe that reason, or maybe add another bool var similar to
the 'check_columnlist' one for this.
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-09 23:45 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-08-09 23:45 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are some more review comments for the HEAD_v8 patch:
======
1. Commit message
If there are two publications, one of them publish a parent table with
(publish_via_partition_root = true) and another publish child table,
subscribing to both publications from one subscription results in two initial
replications. It should only be copied once.
~
I took a 2nd look at that commit message and it seemed slightly
backwards to me - e.g. don't you really mean for the
'publish_via_partition_root' parameter to be used when publishing the
*child* table?
SUGGESTION
If there are two publications, one of them publishing a parent table
directly, and the other publishing a child table with
publish_via_partition_root = true, then subscribing to both those
publications from one subscription results in two initial
replications. It should only be copied once.
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-30 02:24 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-08-30 02:24 UTC (permalink / raw)
To: Peter Smith <[email protected]>; [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wednesday, August 10, 2022 7:45 AM Peter Smith <[email protected]> wrote:
>
> Here are some more review comments for the HEAD_v8 patch:
>
> ======
>
> 1. Commit message
>
> If there are two publications, one of them publish a parent table with
> (publish_via_partition_root = true) and another publish child table, subscribing
> to both publications from one subscription results in two initial replications. It
> should only be copied once.
>
> ~
>
> I took a 2nd look at that commit message and it seemed slightly backwards to
> me - e.g. don't you really mean for the 'publish_via_partition_root' parameter
> to be used when publishing the
> *child* table?
I'm not sure about this, I think we are trying to fix the bug when
'publish_via_partition_root' is used when publishing the parent table.
For this case(via_root used when publishing parent):
CREATE PUBLICATION pub1 for TABLE parent with(publish_via_partition_root);
CREATE PUBLICATION pub2 for TABLE child;
CREATE SUBSCRIPTION sub connect xxx PUBLICATION pub1,pub2;
The expected behavior is only the parent table is published, all the changes
should be replicated using the parent table's identity. So, we should only do
initial sync for the parent table once, but we currently will do table sync for
both parent and child which we think is a bug.
For another case you mentioned(via_root used when publishing child)
CREATE PUBLICATION pub1 for TABLE parent;
CREATE PUBLICATION pub2 for TABLE child with (publish_via_partition_root);
CREATE SUBSCRIPTION sub connect xxx PUBLICATION pub1,pub2;
The expected behavior is only the child table is published, all the changes
should be replicated using the child table's identity. We should do table sync
only for child tables and is same as the current behavior on HEAD. So, I think
there is no bug in this case.
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-08-30 07:43 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-08-30 07:43 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tues, Aug 9, 2022 at 15:15 PM Peter Smith <[email protected]> wrote:
> Here are some review comment for the HEAD_v8 patch:
Thanks for your comments.
> 1. Commit message
>
> If there are two publications, one of them publish a parent table with
> (publish_via_partition_root = true) and another publish child table,
> subscribing to both publications from one subscription results in two initial
> replications. It should only be copied once.
>
> SUGGESTION (Note**)
> If there are two publications - one of them publishing a parent table
> (using publish_via_partition_root = true) and the other is publishing
> one of the parent's child tables - then subscribing to both
> publications from one subscription results in the same initial child
> data being copied twice. It should only be copied once.
>
>
> Note** - I've only reworded the original commit message slightly but
> otherwise left it saying the same thing. But I still have doubts if
> this message actually covers all the cases the patch is trying to
> address. e.g. The comment (see below) in the 'fetch_table_list'
> function seemed to cover more cases than what this commit message is
> describing.
> /*
> * Get the list of tables from publisher, the partition table whose
> * ancestor is also in this list will be ignored, otherwise the initial
> * data in the partition table would be replicated twice.
> */
=> commit message
Changed.
=> Note**
I think the commit message and the comment you mentioned refer to the same kind
of scenario.
> 2. src/backend/catalog/pg_publication.c - pg_get_publication_tables
>
> 2a.
> /*
> - * Returns information of tables in a publication.
> + * Returns information of the tables in the given publication array.
> */
>
> What does "information of the tables" actually mean? Is it just the
> names of the tables; is it more than that? IMO a longer, more
> explanatory comment will be better here instead of a brief ambiguous
> one.
Changed as below:
```
Get information of the tables in the given publication array.
Returns the oid, column list, row filter for each table.
```
> 2b.
> + *results = NIL;
>
> This variable name 'results' is too generic, so it is not helpful when
> trying to understand the subsequent code logic. Please give this a
> meaningful name/description.
Changed the variable name as below:
results -> table_infos
> 2c.
> /* Deconstruct the parameter into elements. */
>
> SUGGESTION
> Deconstruct the parameter into elements where each element is a
> publication name.
Changed.
> 2d.
> + List *current_tables = NIL;
>
> I think this is the tables only on the current pub_elem, so I thought
> 'pub_elem_tables' might make it easier to understand this list's
> meaning.
Changed.
> 2e.
> + /* Now sort and de-duplicate the result list */
> + list_sort(tables, list_oid_cmp);
> + list_deduplicate_oid(tables);
>
> IMO this comment is confusing because there is another list that is
> called the 'results' list, but that is not the same list you are
> processing here. Also, it does not really add anything helpful to just
> have comments saying almost the same as the function names
> (sort/de-duplicate), so please give longer comments to say the reason
> *why* the logic does this rather than just describing the steps.
Fixed the comment. ("result" -> "tables")
I think the purpose of these two functions is clear. The reason I added the
comment here is for consistency with other calling locations.
> 2f.
> + /* Filter by final published table */
> + foreach(lc, results)
>
> Please make this comment more descriptive to explain why/what the
> logic is doing.
Changed as below:
```
For tables that have been filtered out, delete the corresponding
table information in the table_infos list.
```
> 3. src/backend/commands/subscriptioncmds.c - fetch_table_list
>
> 3a.
> + bool check_columnlist = (server_version >= 150000);
>
> Given the assignment, maybe 'columnlist_supported' is a better name?
I am not sure if this name could be changed in this thread.
> 3b.
> + /* Get information of the tables belonging to the specified publications */
>
> For "Get information of..." can you elaborate *what* table
> information this is getting and why?
I am not sure if we need to add a reason.
So, I only added what information we are going to get:
```
Get namespace, relname and column list (if supported) of the tables
belonging to the specified publications.
```
> 3c.
> + if (server_version >= 160000)
> + appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
> + " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
> + " THEN NULL ELSE GPT.attrs END\n"
> + " ) AS attnames\n"
> + " FROM pg_class C\n"
> + " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
> + " JOIN ( SELECT (pg_get_publication_tables(VARIADIC
> array_agg(pubname::text))).*\n"
> + " FROM pg_publication\n"
> + " WHERE pubname IN ( %s )) as GPT\n"
> + " ON GPT.relid = C.oid\n",
> + pub_names.data);
>
> AFAICT the main reason for this check was to decide if you can use the
> new version of 'pg_get_publication_tables' that supports the VARIADIC
> array of pub names or not. If that is correct, then maybe the comment
> should describe that reason, or maybe add another bool var similar to
> the 'check_columnlist' one for this.
I added the comment as following:
```
From version 16, the parameter of the function pg_get_publication_tables
can be an array of publications. The partition table whose ancestor is
also published in this publication array will be filtered out in this
function.
```
I also rebased the REL_15 patch based on the commit (15014b8), and made some
changes to the back-branch patches based on Peter's suggestions.
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v9-0001-Fix-data-replicated-twice-when-specifying-publish.patch (18.6K, ../../OS3PR01MB62752A88C41234061730EC409E799@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v9-0001-Fix-data-replicated-twice-when-specifying-publish.patch)
download | inline diff:
From 5d6265fb1be75d89d637aca97e9a769c508b1989 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v9] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 144 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 75 +++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 193 insertions(+), 76 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 6af3570005..dc017ca9aa 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,29 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns the oid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1056,94 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, If publish_via_partition_root is true, get the
+ * partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = pub_elem->oid;
+ table_infos = lappend(table_infos, result);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ Oid *table_info_tmp = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info_tmp[0]))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1156,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ Oid *table_info = (Oid *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1118,13 +1180,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 670b219c8d..c356624113 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1794,25 +1794,78 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * From version 16, the parameter of the function pg_get_publication_tables
+ * can be an array of publications. The partition table whose ancestor is
+ * also published in this publication array will be filtered out in this
+ * function.
+ */
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index be47583122..16d43156ce 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11681,11 +11681,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables belonging to the specified publications.',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7ec3d2688f..54f5ce5da9 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1449,7 +1449,7 @@ pg_publication_tables| SELECT p.pubname,
JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index f5f8a67092..e0d388d37f 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index b6644556cf..9f88b9ef52 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.23.0.windows.1
[application/octet-stream] REL15_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch (8.4K, ../../OS3PR01MB62752A88C41234061730EC409E799@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL14_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.3K, ../../OS3PR01MB62752A88C41234061730EC409E799@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
[application/octet-stream] REL13_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch (5.3K, ../../OS3PR01MB62752A88C41234061730EC409E799@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v9-0001-Fix-data-replicated-twice-when-specifying-publish_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-19 06:51 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-09-19 06:51 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
FYI, I'm not sure why the cfbot hasn't reported this, but the apply v9
patch failed for me on HEAD as below:
[postgres@CentOS7-x64 oss_postgres_misc]$ git apply
../patches_misc/HEAD_v9-0001-Fix-data-replicated-twice-when-specifying-publish.patch
--verbose
Checking patch src/backend/catalog/pg_publication.c...
Checking patch src/backend/commands/subscriptioncmds.c...
Hunk #1 succeeded at 1917 (offset 123 lines).
Checking patch src/include/catalog/pg_proc.dat...
Hunk #1 succeeded at 11607 (offset -74 lines).
Checking patch src/test/regress/expected/rules.out...
error: while searching for:
JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND
(a.attnum = k.k))))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid,
attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
error: patch failed: src/test/regress/expected/rules.out:1449
error: src/test/regress/expected/rules.out: patch does not apply
Checking patch src/test/subscription/t/013_partition.pl...
Checking patch src/test/subscription/t/028_row_filter.pl...
Checking patch src/test/subscription/t/031_column_list.pl...
------
Kind Regards,
Peter Smith.
Fujitsu Australia.
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-20 06:17 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-09-20 06:17 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Mon, Sept 19, 2022 at 14:52 PM Peter Smith <[email protected]> wrote:
> FYI, I'm not sure why the cfbot hasn't reported this, but the apply v9
> patch failed for me on HEAD as below:
>
> [postgres@CentOS7-x64 oss_postgres_misc]$ git apply
> ../patches_misc/HEAD_v9-0001-Fix-data-replicated-twice-when-specifying-
> publish.patch
> --verbose
> Checking patch src/backend/catalog/pg_publication.c...
> Checking patch src/backend/commands/subscriptioncmds.c...
> Hunk #1 succeeded at 1917 (offset 123 lines).
> Checking patch src/include/catalog/pg_proc.dat...
> Hunk #1 succeeded at 11607 (offset -74 lines).
> Checking patch src/test/regress/expected/rules.out...
> error: while searching for:
> JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND
> (a.attnum = k.k))))) AS attnames,
> pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
> FROM pg_publication p,
> LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid,
> attrs, qual),
> (pg_class c
> JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
> WHERE (c.oid = gpt.relid);
>
> error: patch failed: src/test/regress/expected/rules.out:1449
> error: src/test/regress/expected/rules.out: patch does not apply
> Checking patch src/test/subscription/t/013_partition.pl...
> Checking patch src/test/subscription/t/028_row_filter.pl...
> Checking patch src/test/subscription/t/031_column_list.pl...
Thanks for your kindly reminder.
Rebased the patch based on the changes in HEAD (20b6847).
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v10-0001-Fix-data-replicated-twice-when-specifying-publis.patch (18.7K, ../../OS3PR01MB6275B4F7F5F5CF3A1A76D0FB9E4C9@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v10-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From 18852f8f1483c3eeb18baff7172859edaf8603cc Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v10] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 144 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 75 +++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 193 insertions(+), 76 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 6af3570005..dc017ca9aa 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,29 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns the oid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1056,94 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, If publish_via_partition_root is true, get the
+ * partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = pub_elem->oid;
+ table_infos = lappend(table_infos, result);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ Oid *table_info_tmp = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info_tmp[0]))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1156,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ Oid *table_info = (Oid *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1118,13 +1180,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d042abe341..b825bf7dc8 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1917,25 +1917,78 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * From version 16, the parameter of the function pg_get_publication_tables
+ * can be an array of publications. The partition table whose ancestor is
+ * also published in this publication array will be filtered out in this
+ * function.
+ */
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a07e737a33..ee82d6ee7b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11607,11 +11607,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables belonging to the specified publications.',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..b90c629c3a 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1444,7 +1444,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index f5f8a67092..e0d388d37f 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index b6644556cf..9f88b9ef52 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.23.0.windows.1
[application/octet-stream] REL15_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch (8.4K, ../../OS3PR01MB6275B4F7F5F5CF3A1A76D0FB9E4C9@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL14_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275B4F7F5F5CF3A1A76D0FB9E4C9@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL13_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275B4F7F5F5CF3A1A76D0FB9E4C9@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v10-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-26 02:31 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-09-26 02:31 UTC (permalink / raw)
To: [email protected] <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tuesday, September 20, 2022 3:18 PM Wang, Wei/王 威 <[email protected]> wrote:
> Rebased the patch based on the changes in HEAD (20b6847).
> Attach the new patches.
Hi, thank you for updating the patchset.
FYI, I noticed that the patch for head is no longer applicable.
$ git apply --check HEAD_v10-0001-Fix-data-replicated-twice-when-specifying-publis.patch
error: patch failed: src/backend/catalog/pg_publication.c:1097
error: src/backend/catalog/pg_publication.c: patch does not apply
Also, one minor comment on the change in src/include/catalog/pg_proc.dat.
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables belonging to the specified publications.',
Please remove the period at the end of 'descr' string.
It seems we don't write it in this file and removing it makes the code more aligned.
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-26 04:44 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-09-26 04:44 UTC (permalink / raw)
To: [email protected] <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Mon, Sep 26, 2022 at 10:31 AM Osumi, Takamichi/大墨 昂道 <[email protected]> wrote:
> Hi, thank you for updating the patchset.
>
>
> FYI, I noticed that the patch for head is no longer applicable.
Thanks for your kindly reminder and comment.
> $ git apply --check HEAD_v10-0001-Fix-data-replicated-twice-when-specifying-
> publis.patch
> error: patch failed: src/backend/catalog/pg_publication.c:1097
> error: src/backend/catalog/pg_publication.c: patch does not apply
Rebased the patch based on the changes in HEAD (13a185f).
> Also, one minor comment on the change in src/include/catalog/pg_proc.dat.
>
> # publications
> -{ oid => '6119', descr => 'get information of tables in a publication',
> - proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
> - provolatile => 's', prorettype => 'record', proargtypes => 'text',
> - proallargtypes => '{text,oid,int2vector,pg_node_tree}',
> - proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
> +{ oid => '6119',
> + descr => 'get information of the tables belonging to the specified
> publications.',
>
> Please remove the period at the end of 'descr' string.
> It seems we don't write it in this file and removing it makes the code more
> aligned.
Improved as suggested.
Also modified the description to be consistent with the comments atop the
function pg_get_publication_tables.
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v11-0001-Fix-data-replicated-twice-when-specifying-publis.patch (19.1K, ../../OS3PR01MB62751414E4DABF144F5E481C9E529@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v11-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From b08937a4bebe1ec182282c23bd226deef3005209 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v11] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 148 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 75 +++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 195 insertions(+), 78 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 59967098b3..2996f2d146 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,29 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns the oid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
- FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ FuncCallContext *funcctx;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1056,94 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, If publish_via_partition_root is true, get the
+ * partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = pub_elem->oid;
+ table_infos = lappend(table_infos, result);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ Oid *table_info_tmp = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info_tmp[0]))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1156,23 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ Oid *table_info = (Oid *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info[0],
+ pubid = table_info[1];
Oid schemaid = get_rel_namespace(relid);
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1119,7 +1181,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(pubid);
values[0] = ObjectIdGetDatum(relid);
@@ -1127,13 +1189,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* We don't consider row filters or column lists for FOR ALL TABLES or
* FOR TABLES IN SCHEMA publications.
*/
- if (!publication->alltables &&
+ if (!pub->alltables &&
!SearchSysCacheExists2(PUBLICATIONNAMESPACEMAP,
ObjectIdGetDatum(schemaid),
- ObjectIdGetDatum(publication->oid)))
+ ObjectIdGetDatum(pub->oid)))
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f3bfcca434..6563123bfe 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1915,25 +1915,78 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
- Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
+
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * From version 16, the parameter of the function pg_get_publication_tables
+ * can be an array of publications. The partition table whose ancestor is
+ * also published in this publication array will be filtered out in this
+ * function.
+ */
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ else
+ {
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ",( CASE WHEN (array_length(GPT.attrs, 1) = C.relnatts)\n"
+ " THEN NULL ELSE GPT.attrs END\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a07e737a33..2f0db91a4c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11607,11 +11607,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables in the given publication array',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..b90c629c3a 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1444,7 +1444,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 3e4bfc2178..fb05d01fd0 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.23.0.windows.1
[application/octet-stream] REL15_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch (8.4K, ../../OS3PR01MB62751414E4DABF144F5E481C9E529@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL14_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB62751414E4DABF144F5E481C9E529@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL13_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB62751414E4DABF144F5E481C9E529@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v11-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-27 08:44 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-09-27 08:44 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are my review comments for the HEAD_v11-0001 patch:
======
1. General - Another related bug?
In [1] Hou-san wrote:
For another case you mentioned (via_root used when publishing child)
CREATE PUBLICATION pub1 for TABLE parent;
CREATE PUBLICATION pub2 for TABLE child with (publish_via_partition_root);
CREATE SUBSCRIPTION sub connect xxx PUBLICATION pub1,pub2;
The expected behavior is only the child table is published, all the changes
should be replicated using the child table's identity. We should do table sync
only for child tables and is same as the current behavior on HEAD. So, I think
there is no bug in this case.
~
That behaviour seems different to my understanding because the pgdocs
says when the via_root param is true the 'child' table would be using
the 'parent' identity:
[2] publish_via_partition_root determines whether changes in a
partitioned table (or on its partitions) contained in the publication
will be published using the identity and schema of the partitioned
table rather than that of the individual partitions that are actually
changed.
~
So is this another bug (slightly different from the current one being
patched), or is it just some different special behaviour? If it's
another bug then you need to know that ASAP because I think you may
want to fix both of them at the same time which might impact how this
2x data copy patch should be implemented.
Or perhaps just the pgdocs need more notes about special
cases/combinations like this?
======
2. General - documentation?
For this current patch, IIUC it was decided that it is a bug because
the data gets duplicated, and then some sensible rule was decided that
this patch should use to address it (e.g. publishing a child combined
with publishing a parent via_root will just ignore the child's
publication...).
So my question is - is this (new/fixed) behaviour something that a
user will be able to figure out themselves from the existing
documentation, or does this patch now need its own special notes in
the documentation?
======
3. src/backend/catalog/pg_publication.c - pg_get_publication_tables
+ foreach(lc, pub_elem_tables)
+ {
+ Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
+
+ result[0] = lfirst_oid(lc);
+ result[1] = pub_elem->oid;
+ table_infos = lappend(table_infos, result);
+ }
3a.
It looks like each element in the table_infos list is a malloced obj
of 2x Oids (Oid of table, Oid of pub). IMO better to call this element
'table_info' instead of the meaningless 'result'
~
3b.
Actually, I think it would be better if this function defines a little
2-element structure {Oid relid, Oid pubid} to use instead of this
array (which requires knowledge that [0] means relid and [1] means
pubid).
~~~
4.
+ foreach(lc, table_infos)
+ {
+ Oid *table_info_tmp = (Oid *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info_tmp[0]))
+ table_infos = foreach_delete_current(table_infos, lc);
}
I think the '_tmp' suffix is not helpful here - IMO having another
relid variable would make this more self-explanatory.
Or better yet adopt the suggestion o f #3b and have a little struct
with self-explanatory member names.
=====
5. src/backend/commands/subscriptioncmds.c - fetch_table_list
+ if (server_version >= 160000)
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
Since there is an else statement block, I think this would be more
readable if there was a statement block here too. YMMV
SUGGESTION
if (server_version >= 160000)
{
appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
...
}
~~~
6.
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
6a.
"from publisher, the partition" -> "from the publisher. The partition"
~
6b.
This looks like a common comment that also applied to the "if" part,
so it seems more appropriate to move it to where I indicated below.
Perhaps the whole comment needs a bit of massaging after you move
it...
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * HERE <<<<<<<<<
+ *
+ * From version 16, the parameter of the function pg_get_publication_tables
+ * can be an array of publications. The partition table whose ancestor is
+ * also published in this publication array will be filtered out in this
+ * function.
+ */
------
[1] https://www.postgresql.org/message-id/OS0PR01MB5716A30DDEECC59132E1084F94799%40OS0PR01MB5716.jpnprd0...
[2] https://www.postgresql.org/docs/devel/sql-createpublication.html
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-09-28 08:35 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 2 replies; 99+ messages in thread
From: [email protected] @ 2022-09-28 08:35 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tues, Sep 27, 2022 at 16:45 PM Peter Smith <[email protected]> wrote:
> Here are my review comments for the HEAD_v11-0001 patch:
Thanks for your comments.
> ======
>
> 1. General - Another related bug?
>
> In [1] Hou-san wrote:
>
> For another case you mentioned (via_root used when publishing child)
> CREATE PUBLICATION pub1 for TABLE parent;
> CREATE PUBLICATION pub2 for TABLE child with (publish_via_partition_root);
> CREATE SUBSCRIPTION sub connect xxx PUBLICATION pub1,pub2;
>
> The expected behavior is only the child table is published, all the changes
> should be replicated using the child table's identity. We should do table sync
> only for child tables and is same as the current behavior on HEAD. So, I think
> there is no bug in this case.
>
> ~
>
> That behaviour seems different to my understanding because the pgdocs
> says when the via_root param is true the 'child' table would be using
> the 'parent' identity:
>
> [2] publish_via_partition_root determines whether changes in a
> partitioned table (or on its partitions) contained in the publication
> will be published using the identity and schema of the partitioned
> table rather than that of the individual partitions that are actually
> changed.
>
> ~
>
> So is this another bug (slightly different from the current one being
> patched), or is it just some different special behaviour? If it's
> another bug then you need to know that ASAP because I think you may
> want to fix both of them at the same time which might impact how this
> 2x data copy patch should be implemented.
>
> Or perhaps just the pgdocs need more notes about special
> cases/combinations like this?
>
> ======
>
> 2. General - documentation?
>
> For this current patch, IIUC it was decided that it is a bug because
> the data gets duplicated, and then some sensible rule was decided that
> this patch should use to address it (e.g. publishing a child combined
> with publishing a parent via_root will just ignore the child's
> publication...).
>
> So my question is - is this (new/fixed) behaviour something that a
> user will be able to figure out themselves from the existing
> documentation, or does this patch now need its own special notes in
> the documentation?
IMO this behaviour doesn't look like a bug.
I think the behaviour of multiple publications with parameter
publish_via_partition_root could be added to the pg-doc later in a separate
patch.
> ======
>
> 3. src/backend/catalog/pg_publication.c - pg_get_publication_tables
>
> + foreach(lc, pub_elem_tables)
> + {
> + Oid *result = (Oid *) malloc(sizeof(Oid) * 2);
> +
> + result[0] = lfirst_oid(lc);
> + result[1] = pub_elem->oid;
> + table_infos = lappend(table_infos, result);
> + }
>
> 3a.
> It looks like each element in the table_infos list is a malloced obj
> of 2x Oids (Oid of table, Oid of pub). IMO better to call this element
> 'table_info' instead of the meaningless 'result'
>
> ~
>
> 3b.
> Actually, I think it would be better if this function defines a little
> 2-element structure {Oid relid, Oid pubid} to use instead of this
> array (which requires knowledge that [0] means relid and [1] means
> pubid).
>
> ~~~
>
> 4.
>
> + foreach(lc, table_infos)
> + {
> + Oid *table_info_tmp = (Oid *) lfirst(lc);
> +
> + if (!list_member_oid(tables, table_info_tmp[0]))
> + table_infos = foreach_delete_current(table_infos, lc);
> }
> I think the '_tmp' suffix is not helpful here - IMO having another
> relid variable would make this more self-explanatory.
>
> Or better yet adopt the suggestion o f #3b and have a little struct
> with self-explanatory member names.
Improved as suggested.
> =====
>
> 5. src/backend/commands/subscriptioncmds.c - fetch_table_list
>
> + if (server_version >= 160000)
> + appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
>
> Since there is an else statement block, I think this would be more
> readable if there was a statement block here too. YMMV
>
> SUGGESTION
> if (server_version >= 160000)
> {
> appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
> ...
> }
Improved as suggested.
> ~~~
>
> 6.
>
> + /*
> + * Get the list of tables from publisher, the partition table whose
> + * ancestor is also in this list will be ignored, otherwise the initial
> + * data in the partition table would be replicated twice.
> + */
>
> 6a.
> "from publisher, the partition" -> "from the publisher. The partition"
>
> ~
>
> 6b.
> This looks like a common comment that also applied to the "if" part,
> so it seems more appropriate to move it to where I indicated below.
> Perhaps the whole comment needs a bit of massaging after you move
> it...
>
> + /*
> + * Get namespace, relname and column list (if supported) of the tables
> + * belonging to the specified publications.
> + *
> + * HERE <<<<<<<<<
> + *
> + * From version 16, the parameter of the function pg_get_publication_tables
> + * can be an array of publications. The partition table whose ancestor is
> + * also published in this publication array will be filtered out in this
> + * function.
> + */
Improved as suggested.
Also rebased the patch because the change in the HEAD (20b6847).
Attach the new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v12-0001-Fix-data-replicated-twice-when-specifying-publis.patch (19.7K, ../../OS3PR01MB6275A9B8C65C381C6828DF9D9E549@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v12-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From 6af8bede48777b3acd9ed3c5d63bbc4b6f79f094 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v12] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 151 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 81 +++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
7 files changed, 205 insertions(+), 77 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 59967098b3..830f2b7537 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,35 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns the oid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
+ typedef struct
+ {
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication that publishes this
+ * table. */
+ } published_rel;
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1062,94 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need
+ * those. Otherwise, If publish_via_partition_root is true, get
+ * the partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info = (published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ published_rel *table_info = (published_rel *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info->relid))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1162,22 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ published_rel *table_info = (published_rel *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info->relid;
Oid schemaid = get_rel_namespace(relid);
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1119,7 +1186,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(table_info->pubid);
values[0] = ObjectIdGetDatum(relid);
@@ -1127,13 +1194,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* We don't consider row filters or column lists for FOR ALL TABLES or
* FOR TABLES IN SCHEMA publications.
*/
- if (!publication->alltables &&
+ if (!pub->alltables &&
!SearchSysCacheExists2(PUBLICATIONNAMESPACEMAP,
ObjectIdGetDatum(schemaid),
- ObjectIdGetDatum(publication->oid)))
+ ObjectIdGetDatum(pub->oid)))
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f3bfcca434..c7630872ab 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1915,25 +1915,84 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ initStringInfo(&cmd);
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * Get the list of tables from the publisher. The partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ *
+ * From version 16, the parameter of the function
+ * pg_get_publication_tables can be an array of publications. The
+ * partition table whose ancestor is also published in this publication
+ * array will be filtered out in this function.
+ */
+ if (server_version >= 160000)
+ {
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ }
+ else
+ {
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 8b72f8a215..80bb1408c3 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11607,11 +11607,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables in the given publication array',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..b90c629c3a 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1444,7 +1444,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 3e4bfc2178..fb05d01fd0 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.31.1
[application/octet-stream] REL15_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch (8.6K, ../../OS3PR01MB6275A9B8C65C381C6828DF9D9E549@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-REL15_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL14_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275A9B8C65C381C6828DF9D9E549@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL13_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275A9B8C65C381C6828DF9D9E549@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL13_v12-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-05 03:17 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-10-05 03:17 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
======
1. Missing documentation.
In [1] you wrote:
> I think the behaviour of multiple publications with parameter publish_via_partition_root could be added to the pg-doc later in a separate patch.
~
That doesn't seem right to me. IMO the related documentation updates
cannot really be separated from this patch. Otherwise, what's the
alternative? Push this change, and then (while waiting for the
documentation patch) users will just have to use trial and error to
guess how it works...?
------
2. src/backend/catalog/pg_publication.c
+ typedef struct
+ {
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication that publishes this
+ * table. */
+ } published_rel;
2a.
I think that should be added to typedefs.list
~
2b.
Maybe this also needs some comment to clarify that there will be
*multiple* of these structures in scenarios where the same table is
published by different publications in the array passed.
------
3. QUESTION - pg_get_publication_tables / fetch_table_list
When the same table is published by different publications (but there
are other differences like row-filters/column-lists in each
publication) the result tuple of this function does not include the
pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
as-is but how does it manage to associate each table with the correct
tuple?
I know it apparently all seems to work but I’m not how does that
happen? Can you explain why a puboid is not needed for the result
tuple of this function?
~~
test_pub=# create table t1(a int, b int, c int);
CREATE TABLE
test_pub=# create publication pub1 for table t1(a) where (a > 99);
CREATE PUBLICATION
test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
CREATE PUBLICATION
Following seems OK when I swap orders of publication names...
test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
relid | attrs | rowfilter
-------+-------+-----------
16385 | 1 2 | (b < 33)
16385 | 1 | (a > 99)
(2 rows)
test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
relid | attrs | rowfilter
-------+-------+-----------
16385 | 1 | (a > 99)
16385 | 1 2 | (b < 33)
(2 rows)
But what about this (this is similar to the SQL fragment from
fetch_table_list); I swapped the pub names but the results are the
same...
test_pub=# SELECT pg_get_publication_tables(VARIADIC
array_agg(p.pubname)) from pg_publication p where pubname
IN('pub2','pub1');
pg_get_publication_tables
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------
(16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
:vartype 23 :vartypmod -1 :var
collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
{CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
:constbyval true :constisnull false :
location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
(16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
:opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
:varattno 2 :vartype 23 :vartypmod -1 :v
arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
{CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
:constbyval true :constisnull false
:location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
(2 rows)
test_pub=# SELECT pg_get_publication_tables(VARIADIC
array_agg(p.pubname)) from pg_publication p where pubname
IN('pub1','pub2');
pg_get_publication_tables
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------
(16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
:vartype 23 :vartypmod -1 :var
collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
{CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
:constbyval true :constisnull false :
location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
(16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
:opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
:varattno 2 :vartype 23 :vartypmod -1 :v
arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
{CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
:constbyval true :constisnull false
:location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
(2 rows)
------
[1] https://www.postgresql.org/message-id/OS3PR01MB6275A9B8C65C381C6828DF9D9E549%40OS3PR01MB6275.jpnprd0...
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-05 15:04 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
1 sibling, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-10-05 15:04 UTC (permalink / raw)
To: [email protected] <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wednesday, September 28, 2022 5:36 PM Wang, Wei/王 威 <[email protected]> wrote:
> Also rebased the patch because the change in the HEAD (20b6847).
>
> Attach the new patches.
Hi, thank you for the updated patches!
Here are my minor review comments for HEAD v12.
(1) typo & suggestion to reword one comment
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need
+ * those. Otherwise, If publish_via_partition_root is true, get
+ * the partitioned table itself.
The last sentence has "If" in the middle of the sentence.
We can use the lower letter for it. Or, I think "Otherwise" by itself means
"If publish_via_partition_root is true". So, I'll suggest a below change.
FROM:
Otherwise, If publish_via_partition_root is true, get the partitioned table itself.
TO:
Otherwise, get the partitioned table itself.
(2) Do we need to get "attnames" column from the publisher in the fetch_table_list() ?
When I was looking at v16 path, I didn't see any codes that utilize
the "attnames" column information returned from the publisher.
If we don't need it, could we remove it ?
I can miss something greatly, but this might be affected by HEAD codes ?
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-17 05:49 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 4 replies; 99+ messages in thread
From: [email protected] @ 2022-10-17 05:49 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wed, Oct 5, 2022 at 11:08 AM Peter Smith <[email protected]> wrote:
> Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
Thanks for your comments.
> ======
>
> 1. Missing documentation.
>
> In [1] you wrote:
> > I think the behaviour of multiple publications with parameter
> publish_via_partition_root could be added to the pg-doc later in a separate
> patch.
>
> ~
>
> That doesn't seem right to me. IMO the related documentation updates
> cannot really be separated from this patch. Otherwise, what's the
> alternative? Push this change, and then (while waiting for the
> documentation patch) users will just have to use trial and error to
> guess how it works...?
I tried to add related documentation in a separate patch (HEAD_v13-0002*).
> ------
>
> 2. src/backend/catalog/pg_publication.c
>
> + typedef struct
> + {
> + Oid relid; /* OID of published table */
> + Oid pubid; /* OID of publication that publishes this
> + * table. */
> + } published_rel;
>
> 2a.
> I think that should be added to typedefs.list
Added.
> ~
>
> 2b.
> Maybe this also needs some comment to clarify that there will be
> *multiple* of these structures in scenarios where the same table is
> published by different publications in the array passed.
Added the comments.
> ------
>
> 3. QUESTION - pg_get_publication_tables / fetch_table_list
>
> When the same table is published by different publications (but there
> are other differences like row-filters/column-lists in each
> publication) the result tuple of this function does not include the
> pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
> as-is but how does it manage to associate each table with the correct
> tuple?
>
> I know it apparently all seems to work but I’m not how does that
> happen? Can you explain why a puboid is not needed for the result
> tuple of this function?
Sorry, I am not sure I understand your question.
I try to answer your question by explaining the two functions you mentioned:
First, the function pg_get_publication_tables gets the list (see table_infos)
that included published table and the corresponding publication. Then based
on this list, the function pg_get_publication_tables returns information
(scheme, relname, row filter and column list) about the published tables in the
publications list. It just doesn't return pubid.
Then, the SQL in the function fetch_table_list will get the columns in the
column list from pg_attribute. (This is to return all columns when the column
list is not specified)
> ~~
>
> test_pub=# create table t1(a int, b int, c int);
> CREATE TABLE
> test_pub=# create publication pub1 for table t1(a) where (a > 99);
> CREATE PUBLICATION
> test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
> CREATE PUBLICATION
>
> Following seems OK when I swap orders of publication names...
>
> test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
> relid | attrs | rowfilter
> -------+-------+-----------
> 16385 | 1 2 | (b < 33)
> 16385 | 1 | (a > 99)
> (2 rows)
>
> test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
> relid | attrs | rowfilter
> -------+-------+-----------
> 16385 | 1 | (a > 99)
> 16385 | 1 2 | (b < 33)
> (2 rows)
>
> But what about this (this is similar to the SQL fragment from
> fetch_table_list); I swapped the pub names but the results are the
> same...
>
> test_pub=# SELECT pg_get_publication_tables(VARIADIC
> array_agg(p.pubname)) from pg_publication p where pubname
> IN('pub2','pub1');
>
> pg_get_publication_tables
>
> -------------------------------------------------------------------------------------------------
> ---------------------------------------------------------------------
> -------------------------------------------------------------------------------------------------
> ---------------------------------------------------------------------
> -------------------------------------------------------------------
> (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> :vartype 23 :vartypmod -1 :var
> collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> :constbyval true :constisnull false :
> location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> :varattno 2 :vartype 23 :vartypmod -1 :v
> arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> :constbyval true :constisnull false
> :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> (2 rows)
>
> test_pub=# SELECT pg_get_publication_tables(VARIADIC
> array_agg(p.pubname)) from pg_publication p where pubname
> IN('pub1','pub2');
>
> pg_get_publication_tables
>
> -------------------------------------------------------------------------------------------------
> ---------------------------------------------------------------------
> -------------------------------------------------------------------------------------------------
> ---------------------------------------------------------------------
> -------------------------------------------------------------------
> (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> :vartype 23 :vartypmod -1 :var
> collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> :constbyval true :constisnull false :
> location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> :varattno 2 :vartype 23 :vartypmod -1 :v
> arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> :constbyval true :constisnull false
> :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> (2 rows)
I think this is because the usage of SELECT statement. The order seems depend
on pg_publication. Such as:
postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub1','pub2');
array_agg
-------------
{pub1,pub2}
(1 row)
postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub2','pub1');
array_agg
-------------
{pub1,pub2}
(1 row)
Attach the new patch set.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch (20.2K, ../../OS3PR01MB6275843B2BBE92870F7881C19E299@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From 098ce6c3f52a6f06a39f1dadfa505fa092ac5fa2 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v13 1/2] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 154 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 81 +++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
src/tools/pgindent/typedefs.list | 1 +
8 files changed, 209 insertions(+), 77 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 59967098b3..b46abce4a0 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,35 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns the oid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
#define NUM_PUBLICATION_TABLES_ELEM 3
+ typedef struct
+ {
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication that publishes this
+ * table. */
+ } published_rel;
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,42 +1062,97 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, get the partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ *
+ * When a table is published by multiple publications, to obtain
+ * all row filters and column list, the structure related to this
+ * table will be recorded multiple times.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info = (published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ published_rel *table_info = (published_rel *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info->relid))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
@@ -1097,20 +1165,22 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ published_rel *table_info = (published_rel *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info->relid;
Oid schemaid = get_rel_namespace(relid);
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1119,7 +1189,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(table_info->pubid);
values[0] = ObjectIdGetDatum(relid);
@@ -1127,13 +1197,13 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* We don't consider row filters or column lists for FOR ALL TABLES or
* FOR TABLES IN SCHEMA publications.
*/
- if (!publication->alltables &&
+ if (!pub->alltables &&
!SearchSysCacheExists2(PUBLICATIONNAMESPACEMAP,
ObjectIdGetDatum(schemaid),
- ObjectIdGetDatum(publication->oid)))
+ ObjectIdGetDatum(pub->oid)))
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f3bfcca434..c7630872ab 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1915,25 +1915,84 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ initStringInfo(&cmd);
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * Get the list of tables from the publisher. The partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ *
+ * From version 16, the parameter of the function
+ * pg_get_publication_tables can be an array of publications. The
+ * partition table whose ancestor is also published in this publication
+ * array will be filtered out in this function.
+ */
+ if (server_version >= 160000)
+ {
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ }
+ else
+ {
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 68bb032d3e..73775837b8 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11610,11 +11610,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables in the given publication array',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..b90c629c3a 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1444,7 +1444,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index ae022faa78..a39f3d27ff 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 97c9bc1861..bae80b2235 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3579,6 +3579,7 @@ pthread_mutex_t
pthread_once_t
pthread_t
ptrdiff_t
+published_rel
pull_var_clause_context
pull_varattnos_context
pull_varnos_context
--
2.23.0.windows.1
[application/octet-stream] HEAD_v13-0002-Add-clarification-for-the-behaviour-of-the-publi.patch (2.3K, ../../OS3PR01MB6275843B2BBE92870F7881C19E299@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-HEAD_v13-0002-Add-clarification-for-the-behaviour-of-the-publi.patch)
download | inline diff:
From 1f790afa528780cc528e4940996af5e2a9b15294 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 28 Sep 2022 14:35:41 +0800
Subject: [PATCH v13 2/2] Add clarification for the behaviour of the
publication parameter publish_via_partition_root
The following usage scenarios are not described in detail in the manual:
If one subscription subscribes multiple publications, and these publications
publish a partitioned table and its partitions respectively. When we specify
this parameter on one or more of these publications, which identity and schema
should be used to publish the changes?
In these cases, I think the parameter publish_via_partition_root behave as
follows:
[publisher-side]
create table parent (a int primary key) partition by range (a);
create table child partition of parent default;
create publication pub1 for table parent;
create publication pub2 for table child;
[subscriber-side]
create subscription sub connection 'xxxx' publication pub1, pub2;
- If the parameter publish_via_partition_root is specified only in pub1,
changes will be published using the identity and schema of the table parent.
- If the parameter publish_via_partition_root is specified only in pub2,
changes will be published using the identity and schema of the table child.
---
doc/src/sgml/ref/create_publication.sgml | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml
index c5190f0ce6..a4f84bfa83 100644
--- a/doc/src/sgml/ref/create_publication.sgml
+++ b/doc/src/sgml/ref/create_publication.sgml
@@ -201,6 +201,13 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
consisting of a different set of partitions.
</para>
+ <para>
+ If a root partitioned table is published by any subscribed publications which
+ set publish_via_partition_root = true, changes on this root partitioned table
+ (or on its partitions) will be published using the identity and schema of this
+ root partitioned table rather than that of the individual partitions.
+ </para>
+
<para>
This parameter also affects how row filters and column lists are
chosen for partitions; see below for details.
--
2.23.0.windows.1
[application/octet-stream] REL15_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch (8.6K, ../../OS3PR01MB6275843B2BBE92870F7881C19E299@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL15_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From 189f5418b260e730167268d0044b8072b6b88d51 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 20 Jul 2022 09:47:14 +0800
Subject: [PATCH v13] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, we exclude the partition table whose ancestor belongs to specified
publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 45 ++++++++++++++++++----
src/test/subscription/t/013_partition.pl | 18 ++-------
src/test/subscription/t/028_row_filter.pl | 13 ++++++-
src/test/subscription/t/031_column_list.pl | 5 ++-
4 files changed, 56 insertions(+), 25 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index ef4f9c175c..05b6b4c8a1 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1756,25 +1756,56 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
+
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * date in the partition table would be replicated twice.
+ */
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
/* Get column lists for each relation if the publisher supports it */
if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ appendStringInfoString(&cmd, ", ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index ae022faa78..a39f3d27ff 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
--
2.23.0.windows.1
[application/octet-stream] REL14_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch (5.3K, ../../OS3PR01MB6275843B2BBE92870F7881C19E299@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL14_v13-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From b4ba43aae8fe41faa8a9d10b958e0f1d718bca72 Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Thu, 28 Apr 2022 11:28:35 +0800
Subject: [PATCH v13] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, we exclude the partition table whose ancestor belongs to specified
publications when getting the table list from publisher.
---
src/backend/commands/subscriptioncmds.c | 35 ++++++++++++++++++------
src/test/subscription/t/013_partition.pl | 18 ++++--------
2 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1719f04517..411705f20b 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1477,7 +1477,8 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[2] = {TEXTOID, TEXTOID};
ListCell *lc;
@@ -1486,10 +1487,7 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(list_length(publications) > 0);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
+ initStringInfo(&pub_names);
first = true;
foreach(lc, publications)
{
@@ -1498,14 +1496,35 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
if (first)
first = false;
else
- appendStringInfoString(&cmd, ", ");
+ appendStringInfoString(&pub_names, ", ");
- appendStringInfoString(&cmd, quote_literal_cstr(pubname));
+ appendStringInfoString(&pub_names, quote_literal_cstr(pubname));
}
- appendStringInfoChar(&cmd, ')');
+
+ /*
+ * Get the list of tables from publisher, the partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ */
+ initStringInfo(&cmd);
+ appendStringInfo(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n"
+ " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n"
+ " FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n",
+ pub_names.data);
res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
pfree(cmd.data);
+ pfree(pub_names.data);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 58d78b4292..9d364f41bc 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -6,7 +6,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 71;
+use Test::More tests => 69;
# setup
@@ -467,12 +467,14 @@ $node_subscriber2->safe_psql('postgres',
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
+
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)"
);
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)"
-);
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations.
$node_subscriber2->safe_psql('postgres',
@@ -540,10 +542,6 @@ $result = $node_subscriber2->safe_psql('postgres',
"SELECT a FROM tab4 ORDER BY 1");
is( $result, qq(0), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -567,10 +565,6 @@ $result = $node_subscriber2->safe_psql('postgres',
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result = $node_subscriber2->safe_psql('postgres',
- "SELECT a FROM tab4_1 ORDER BY 1");
-is( $result, qq(), 'inserts into tab4_1 replicated');
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
--
2.23.0.windows.1
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-17 05:52 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-10-17 05:52 UTC (permalink / raw)
To: [email protected] <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Wed, Oct 5, 2022 at 23:05 PM Osumi, Takamichi/大墨 昂道 <[email protected]> wrote:
> Hi, thank you for the updated patches!
>
>
> Here are my minor review comments for HEAD v12.
Thanks for your comments.
> (1) typo & suggestion to reword one comment
>
>
> + * Publications support partitioned tables. If
> + * publish_via_partition_root is false, all changes are replicated
> + * using leaf partition identity and schema, so we only need
> + * those. Otherwise, If publish_via_partition_root is true, get
> + * the partitioned table itself.
>
>
> The last sentence has "If" in the middle of the sentence.
> We can use the lower letter for it. Or, I think "Otherwise" by itself means
> "If publish_via_partition_root is true". So, I'll suggest a below change.
>
>
> FROM:
> Otherwise, If publish_via_partition_root is true, get the partitioned table itself.
> TO:
> Otherwise, get the partitioned table itself.
Improved.
> (2) Do we need to get "attnames" column from the publisher in the
> fetch_table_list() ?
>
> When I was looking at v16 path, I didn't see any codes that utilize
> the "attnames" column information returned from the publisher.
> If we don't need it, could we remove it ?
> I can miss something greatly, but this might be affected by HEAD codes ?
Yes, it is affected by HEAD. I think we need this column to check whether the
same table has multiple column lists. (see commit fd0b9dc)
The new patch set were attached in [1].
[1] - https://www.postgresql.org/message-id/OS3PR01MB6275843B2BBE92870F7881C19E299%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-21 08:52 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
3 siblings, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-10-21 08:52 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Mon, Oct 17, 2022 at 4:49 PM [email protected]
<[email protected]> wrote:
>
> On Wed, Oct 5, 2022 at 11:08 AM Peter Smith <[email protected]> wrote:
> > Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
>
...
> >
> > 3. QUESTION - pg_get_publication_tables / fetch_table_list
> >
> > When the same table is published by different publications (but there
> > are other differences like row-filters/column-lists in each
> > publication) the result tuple of this function does not include the
> > pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
> > as-is but how does it manage to associate each table with the correct
> > tuple?
> >
> > I know it apparently all seems to work but I’m not how does that
> > happen? Can you explain why a puboid is not needed for the result
> > tuple of this function?
>
> Sorry, I am not sure I understand your question.
> I try to answer your question by explaining the two functions you mentioned:
>
> First, the function pg_get_publication_tables gets the list (see table_infos)
> that included published table and the corresponding publication. Then based
> on this list, the function pg_get_publication_tables returns information
> (scheme, relname, row filter and column list) about the published tables in the
> publications list. It just doesn't return pubid.
>
> Then, the SQL in the function fetch_table_list will get the columns in the
> column list from pg_attribute. (This is to return all columns when the column
> list is not specified)
>
I meant, for example, if the different publications specified
different col-lists for the same table then IIUC the
fetch_table_lists() is going to return 2 list elements
(schema,rel_name,row_filter,col_list). But when the schema/rel_name
are the same for 2 elements then (without also a pubid) how are you
going to know where the list element came from, and how come that is
not important?
> > ~~
> >
> > test_pub=# create table t1(a int, b int, c int);
> > CREATE TABLE
> > test_pub=# create publication pub1 for table t1(a) where (a > 99);
> > CREATE PUBLICATION
> > test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
> > CREATE PUBLICATION
> >
> > Following seems OK when I swap orders of publication names...
> >
> > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
> > relid | attrs | rowfilter
> > -------+-------+-----------
> > 16385 | 1 2 | (b < 33)
> > 16385 | 1 | (a > 99)
> > (2 rows)
> >
> > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
> > relid | attrs | rowfilter
> > -------+-------+-----------
> > 16385 | 1 | (a > 99)
> > 16385 | 1 2 | (b < 33)
> > (2 rows)
> >
> > But what about this (this is similar to the SQL fragment from
> > fetch_table_list); I swapped the pub names but the results are the
> > same...
> >
> > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > array_agg(p.pubname)) from pg_publication p where pubname
> > IN('pub2','pub1');
> >
> > pg_get_publication_tables
> >
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------
> > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > :vartype 23 :vartypmod -1 :var
> > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false :
> > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > :varattno 2 :vartype 23 :vartypmod -1 :v
> > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false
> > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > (2 rows)
> >
> > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > array_agg(p.pubname)) from pg_publication p where pubname
> > IN('pub1','pub2');
> >
> > pg_get_publication_tables
> >
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------
> > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > :vartype 23 :vartypmod -1 :var
> > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false :
> > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > :varattno 2 :vartype 23 :vartypmod -1 :v
> > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false
> > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > (2 rows)
>
> I think this is because the usage of SELECT statement. The order seems depend
> on pg_publication. Such as:
>
> postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub1','pub2');
> array_agg
> -------------
> {pub1,pub2}
> (1 row)
>
> postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub2','pub1');
> array_agg
> -------------
> {pub1,pub2}
> (1 row)
>
Right, so I felt it was a bit dubious that the result of the function
“seems to depend on” something. That’s why I was asking why the list
elements did not include a pubid. Then a caller could be certain what
element belonged with what publication. It's not quite clear to me why
that is not important for this patch - but anyway, even if it's not
necessary for this patch's usage, this is a function that is exposed
to users who might have different needs/expectations than this patch
has, so shouldn't the result be less fuzzy for them?
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-10-21 09:01 Peter Smith <[email protected]>
parent: [email protected] <[email protected]>
3 siblings, 1 reply; 99+ messages in thread
From: Peter Smith @ 2022-10-21 09:01 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
Here are my review comments for HEAD patches v13*
//////
Patch HEAD_v13-0001
I already posted some follow-up questions. See [1]
/////
Patch HEAD_v13-0002
1. Commit message
The following usage scenarios are not described in detail in the manual:
If one subscription subscribes multiple publications, and these publications
publish a partitioned table and its partitions respectively. When we specify
this parameter on one or more of these publications, which identity and schema
should be used to publish the changes?
In these cases, I think the parameter publish_via_partition_root behave as
follows:
~
It seemed worded a bit strangely. Also, you said "on one or more of
these publications" but the examples only show only one publication
having 'publish_via_partition_root'.
SUGGESTION (I've modified the wording slightly but the examples are unchanged).
Assume a subscription is subscribing to multiple publications, and
these publications publish a partitioned table and its partitions
respectively:
[publisher-side]
create table parent (a int primary key) partition by range (a);
create table child partition of parent default;
create publication pub1 for table parent;
create publication pub2 for table child;
[subscriber-side]
create subscription sub connection 'xxxx' publication pub1, pub2;
The manual does not clearly describe the behaviour when the user had
specified the parameter 'publish_via_partition_root' on just one of
the publications. This patch modifies documentation to clarify the
following rules:
- If the parameter publish_via_partition_root is specified only in pub1,
changes will be published using the identity and schema of the table 'parent'.
- If the parameter publish_via_partition_root is specified only in pub2,
changes will be published using the identity and schema of the table 'child'.
~~~
2.
- If the parameter publish_via_partition_root is specified only in pub2,
changes will be published using the identity and schema of the table child.
~
Is that right though? This rule seems 100% contrary to the meaning of
'publish_via_partition_root=true'.
------
3. doc/src/sgml/ref/create_publication.sgml
+ <para>
+ If a root partitioned table is published by any subscribed
publications which
+ set publish_via_partition_root = true, changes on this root
partitioned table
+ (or on its partitions) will be published using the identity
and schema of this
+ root partitioned table rather than that of the individual partitions.
+ </para>
This seems to only describe the first example from the commit message.
What about some description to explain the second example?
------
[1] https://www.postgresql.org/message-id/CAHut%2BPt%2B1PNx6VsZ-xKzAU-18HmNXhjCC1TGakKX46Wg7YNT1Q%40mail...
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-03 23:34 Ian Lawrence Barwick <[email protected]>
parent: [email protected] <[email protected]>
3 siblings, 0 replies; 99+ messages in thread
From: Ian Lawrence Barwick @ 2022-11-03 23:34 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; [email protected] <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
2022年10月17日(月) 14:49 [email protected] <[email protected]>:
>
> On Wed, Oct 5, 2022 at 11:08 AM Peter Smith <[email protected]> wrote:
> > Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
>
> Thanks for your comments.
>
> > ======
> >
> > 1. Missing documentation.
> >
> > In [1] you wrote:
> > > I think the behaviour of multiple publications with parameter
> > publish_via_partition_root could be added to the pg-doc later in a separate
> > patch.
> >
> > ~
> >
> > That doesn't seem right to me. IMO the related documentation updates
> > cannot really be separated from this patch. Otherwise, what's the
> > alternative? Push this change, and then (while waiting for the
> > documentation patch) users will just have to use trial and error to
> > guess how it works...?
>
> I tried to add related documentation in a separate patch (HEAD_v13-0002*).
>
> > ------
> >
> > 2. src/backend/catalog/pg_publication.c
> >
> > + typedef struct
> > + {
> > + Oid relid; /* OID of published table */
> > + Oid pubid; /* OID of publication that publishes this
> > + * table. */
> > + } published_rel;
> >
> > 2a.
> > I think that should be added to typedefs.list
>
> Added.
>
> > ~
> >
> > 2b.
> > Maybe this also needs some comment to clarify that there will be
> > *multiple* of these structures in scenarios where the same table is
> > published by different publications in the array passed.
>
> Added the comments.
>
> > ------
> >
> > 3. QUESTION - pg_get_publication_tables / fetch_table_list
> >
> > When the same table is published by different publications (but there
> > are other differences like row-filters/column-lists in each
> > publication) the result tuple of this function does not include the
> > pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
> > as-is but how does it manage to associate each table with the correct
> > tuple?
> >
> > I know it apparently all seems to work but I’m not how does that
> > happen? Can you explain why a puboid is not needed for the result
> > tuple of this function?
>
> Sorry, I am not sure I understand your question.
> I try to answer your question by explaining the two functions you mentioned:
>
> First, the function pg_get_publication_tables gets the list (see table_infos)
> that included published table and the corresponding publication. Then based
> on this list, the function pg_get_publication_tables returns information
> (scheme, relname, row filter and column list) about the published tables in the
> publications list. It just doesn't return pubid.
>
> Then, the SQL in the function fetch_table_list will get the columns in the
> column list from pg_attribute. (This is to return all columns when the column
> list is not specified)
>
> > ~~
> >
> > test_pub=# create table t1(a int, b int, c int);
> > CREATE TABLE
> > test_pub=# create publication pub1 for table t1(a) where (a > 99);
> > CREATE PUBLICATION
> > test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
> > CREATE PUBLICATION
> >
> > Following seems OK when I swap orders of publication names...
> >
> > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
> > relid | attrs | rowfilter
> > -------+-------+-----------
> > 16385 | 1 2 | (b < 33)
> > 16385 | 1 | (a > 99)
> > (2 rows)
> >
> > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
> > relid | attrs | rowfilter
> > -------+-------+-----------
> > 16385 | 1 | (a > 99)
> > 16385 | 1 2 | (b < 33)
> > (2 rows)
> >
> > But what about this (this is similar to the SQL fragment from
> > fetch_table_list); I swapped the pub names but the results are the
> > same...
> >
> > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > array_agg(p.pubname)) from pg_publication p where pubname
> > IN('pub2','pub1');
> >
> > pg_get_publication_tables
> >
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------
> > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > :vartype 23 :vartypmod -1 :var
> > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false :
> > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > :varattno 2 :vartype 23 :vartypmod -1 :v
> > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false
> > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > (2 rows)
> >
> > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > array_agg(p.pubname)) from pg_publication p where pubname
> > IN('pub1','pub2');
> >
> > pg_get_publication_tables
> >
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------------------------------------
> > ---------------------------------------------------------------------
> > -------------------------------------------------------------------
> > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > :vartype 23 :vartypmod -1 :var
> > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false :
> > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > :varattno 2 :vartype 23 :vartypmod -1 :v
> > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > :constbyval true :constisnull false
> > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > (2 rows)
>
> I think this is because the usage of SELECT statement. The order seems depend
> on pg_publication. Such as:
>
> postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub1','pub2');
> array_agg
> -------------
> {pub1,pub2}
> (1 row)
>
> postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE pubname IN ('pub2','pub1');
> array_agg
> -------------
> {pub1,pub2}
> (1 row)
>
> Attach the new patch set.
This entry was marked as "Needs review" in the CommitFest app but cfbot
reports the patch no longer applies.
We've marked it as "Waiting on Author". As CommitFest 2022-11 is
currently underway, this would be an excellent time update the patch.
Once you think the patchset is ready for review again, you (or any
interested party) can move the patch entry forward by visiting
https://commitfest.postgresql.org/40/3623/
and changing the status to "Needs review".
Thanks
Ian Barwick
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-08 04:12 Takamichi Osumi (Fujitsu) <[email protected]>
parent: [email protected] <[email protected]>
3 siblings, 1 reply; 99+ messages in thread
From: Takamichi Osumi (Fujitsu) @ 2022-11-08 04:12 UTC (permalink / raw)
To: [email protected] <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Monday, October 17, 2022 2:49 PM Wang, Wei/王 威 <[email protected]> wrote:
> Attach the new patch set.
Hi, thank you for posting the new patches.
Here are minor comments on the HEAD_v13-0002.
(1) Suggestion for the document description
+ <para>
+ If a root partitioned table is published by any subscribed publications which
+ set publish_via_partition_root = true, changes on this root partitioned table
+ (or on its partitions) will be published using the identity and schema of this
+ root partitioned table rather than that of the individual partitions.
+ </para>
+
I suppose this sentence looks quite similar to the one in the previous paragraph and can be adjusted.
IIUC the main value of the patch is to clarify what happens when
we mix publications of different publish_via_partition_root settings for one partition hierarchy.
If this is true, how about below sentence instead of the one above ?
"
There can be a case where a subscription combines publications with
different publish_via_partition_root values for one same partition hierarchy
(e.g. subscribe two publications indicating the root partitioned table and its child table respectively).
In this case, the identity and schema of the root partitioned table take priority.
"
(2) Better documentation alignment
I think we need to wrap publish_via_partition_root by "literal" tag
in the documentation create_publication.sgml.
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-11 05:43 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-11-11 05:43 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, Oct 21, 2022 at 17:02 PM Peter Smith <[email protected]> wrote:
>
Thanks for your comments. Sorry for not replying in time.
> On Mon, Oct 17, 2022 at 4:49 PM [email protected]
> <[email protected]> wrote:
> >
> > On Wed, Oct 5, 2022 at 11:08 AM Peter Smith <[email protected]>
> wrote:
> > > Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
> >
> ...
> > >
> > > 3. QUESTION - pg_get_publication_tables / fetch_table_list
> > >
> > > When the same table is published by different publications (but there
> > > are other differences like row-filters/column-lists in each
> > > publication) the result tuple of this function does not include the
> > > pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
> > > as-is but how does it manage to associate each table with the correct
> > > tuple?
> > >
> > > I know it apparently all seems to work but I’m not how does that
> > > happen? Can you explain why a puboid is not needed for the result
> > > tuple of this function?
> >
> > Sorry, I am not sure I understand your question.
> > I try to answer your question by explaining the two functions you mentioned:
> >
> > First, the function pg_get_publication_tables gets the list (see table_infos)
> > that included published table and the corresponding publication. Then based
> > on this list, the function pg_get_publication_tables returns information
> > (scheme, relname, row filter and column list) about the published tables in the
> > publications list. It just doesn't return pubid.
> >
> > Then, the SQL in the function fetch_table_list will get the columns in the
> > column list from pg_attribute. (This is to return all columns when the column
> > list is not specified)
> >
>
> I meant, for example, if the different publications specified
> different col-lists for the same table then IIUC the
> fetch_table_lists() is going to return 2 list elements
> (schema,rel_name,row_filter,col_list). But when the schema/rel_name
> are the same for 2 elements then (without also a pubid) how are you
> going to know where the list element came from, and how come that is
> not important?
>
> > > ~~
> > >
> > > test_pub=# create table t1(a int, b int, c int);
> > > CREATE TABLE
> > > test_pub=# create publication pub1 for table t1(a) where (a > 99);
> > > CREATE PUBLICATION
> > > test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
> > > CREATE PUBLICATION
> > >
> > > Following seems OK when I swap orders of publication names...
> > >
> > > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > > ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
> > > relid | attrs | rowfilter
> > > -------+-------+-----------
> > > 16385 | 1 2 | (b < 33)
> > > 16385 | 1 | (a > 99)
> > > (2 rows)
> > >
> > > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > > ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
> > > relid | attrs | rowfilter
> > > -------+-------+-----------
> > > 16385 | 1 | (a > 99)
> > > 16385 | 1 2 | (b < 33)
> > > (2 rows)
> > >
> > > But what about this (this is similar to the SQL fragment from
> > > fetch_table_list); I swapped the pub names but the results are the
> > > same...
> > >
> > > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > > array_agg(p.pubname)) from pg_publication p where pubname
> > > IN('pub2','pub1');
> > >
> > > pg_get_publication_tables
> > >
> > > ---------------------------------------------------------------------------------------------
> ----
> > > ---------------------------------------------------------------------
> > > ---------------------------------------------------------------------------------------------
> ----
> > > ---------------------------------------------------------------------
> > > -------------------------------------------------------------------
> > > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > > :vartype 23 :vartypmod -1 :var
> > > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > :constbyval true :constisnull false :
> > > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > > :varattno 2 :vartype 23 :vartypmod -1 :v
> > > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > :constbyval true :constisnull false
> > > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > > (2 rows)
> > >
> > > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > > array_agg(p.pubname)) from pg_publication p where pubname
> > > IN('pub1','pub2');
> > >
> > > pg_get_publication_tables
> > >
> > > ---------------------------------------------------------------------------------------------
> ----
> > > ---------------------------------------------------------------------
> > > ---------------------------------------------------------------------------------------------
> ----
> > > ---------------------------------------------------------------------
> > > -------------------------------------------------------------------
> > > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > > :vartype 23 :vartypmod -1 :var
> > > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > :constbyval true :constisnull false :
> > > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > > :varattno 2 :vartype 23 :vartypmod -1 :v
> > > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > :constbyval true :constisnull false
> > > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > > (2 rows)
> >
> > I think this is because the usage of SELECT statement. The order seems
> depend
> > on pg_publication. Such as:
> >
> > postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE
> pubname IN ('pub1','pub2');
> > array_agg
> > -------------
> > {pub1,pub2}
> > (1 row)
> >
> > postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE
> pubname IN ('pub2','pub1');
> > array_agg
> > -------------
> > {pub1,pub2}
> > (1 row)
> >
>
> Right, so I felt it was a bit dubious that the result of the function
> “seems to depend on” something. That’s why I was asking why the list
> elements did not include a pubid. Then a caller could be certain what
> element belonged with what publication. It's not quite clear to me why
> that is not important for this patch - but anyway, even if it's not
> necessary for this patch's usage, this is a function that is exposed
> to users who might have different needs/expectations than this patch
> has, so shouldn't the result be less fuzzy for them?
Yes, I agree that there may be such a need in the future. Added 'pubid' to the
output of this function.
BTW, I think the usage of the function pg_get_publication_tables is not
documented in the pg-doc now, it doesn't seem to be a function provided to
users. So I didn't modify the documentation.
Attach new patches.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v14-0001-Fix-data-replicated-twice-when-specifying-publis.patch (21.4K, ../../OS3PR01MB6275FB5397C6A647F262A3A69E009@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v14-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From c343eb4c5bf23c3c565c6a5e73fa541e837afbca Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v14 1/2] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 177 +++++++++++++++------
src/backend/commands/subscriptioncmds.c | 81 ++++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +--
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
src/tools/pgindent/typedefs.list | 1 +
8 files changed, 222 insertions(+), 87 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 59967098b3..1a5dedc1a8 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1026,22 +1026,35 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns pubid, relid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
-#define NUM_PUBLICATION_TABLES_ELEM 3
+#define NUM_PUBLICATION_TABLES_ELEM 4
+ typedef struct
+ {
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication that publishes this
+ * table. */
+ } published_rel;
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *tables = NIL,
+ *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,68 +1062,127 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, get the partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ *
+ * When a table is published by multiple publications, to obtain
+ * all row filters and column list, the structure related to this
+ * table will be recorded multiple times.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info = (published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
+ }
+
+ pfree(elems);
+
+ /* Now sort and de-duplicate the tables list */
+ list_sort(tables, list_oid_cmp);
+ list_deduplicate_oid(tables);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ tables = filter_partitions(tables);
+
+ /*
+ * For tables that have been filtered out, delete the corresponding
+ * table information in the table_infos list.
+ */
+ foreach(lc, table_infos)
+ {
+ published_rel *table_info = (published_rel *) lfirst(lc);
+
+ if (!list_member_oid(tables, table_info->relid))
+ table_infos = foreach_delete_current(table_infos, lc);
}
/* Construct a tuple descriptor for the result rows. */
tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATION_TABLES_ELEM);
- TupleDescInitEntry(tupdesc, (AttrNumber) 1, "relid",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pubid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "relid",
OIDOID, -1, 0);
- TupleDescInitEntry(tupdesc, (AttrNumber) 2, "attrs",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "attrs",
INT2VECTOROID, -1, 0);
- TupleDescInitEntry(tupdesc, (AttrNumber) 3, "qual",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "qual",
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ published_rel *table_info = (published_rel *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info->relid;
Oid schemaid = get_rel_namespace(relid);
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1119,38 +1191,39 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(table_info->pubid);
- values[0] = ObjectIdGetDatum(relid);
+ values[0] = ObjectIdGetDatum(pub->oid);
+ values[1] = ObjectIdGetDatum(relid);
/*
* We don't consider row filters or column lists for FOR ALL TABLES or
* FOR TABLES IN SCHEMA publications.
*/
- if (!publication->alltables &&
+ if (!pub->alltables &&
!SearchSysCacheExists2(PUBLICATIONNAMESPACEMAP,
ObjectIdGetDatum(schemaid),
- ObjectIdGetDatum(publication->oid)))
+ ObjectIdGetDatum(pub->oid)))
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
/* Lookup the column list attribute. */
- values[1] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
Anum_pg_publication_rel_prattrs,
- &(nulls[1]));
+ &(nulls[2]));
/* Null indicates no filter. */
- values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ values[3] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
Anum_pg_publication_rel_prqual,
- &(nulls[2]));
+ &(nulls[3]));
}
else
{
- nulls[1] = true;
nulls[2] = true;
+ nulls[3] = true;
}
rettuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f0cec2ad5e..f3e002d380 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1920,25 +1920,84 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ initStringInfo(&cmd);
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * Get the list of tables from the publisher. The partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ *
+ * From version 16, the parameter of the function
+ * pg_get_publication_tables can be an array of publications. The
+ * partition table whose ancestor is also published in this publication
+ * array will be filtered out in this function.
+ */
+ if (server_version >= 160000)
+ {
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ }
+ else
+ {
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 20f5aa56ea..2b2d45d838 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11615,11 +11615,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables in the given publication array',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o,o}', proargnames => '{pubname,pubid,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..5d595db68b 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1446,7 +1446,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(pubid, relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index ae022faa78..a39f3d27ff 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 245aea1dd1..d64b0f6390 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3581,6 +3581,7 @@ pthread_mutex_t
pthread_once_t
pthread_t
ptrdiff_t
+published_rel
pull_var_clause_context
pull_varattnos_context
pull_varnos_context
--
2.23.0.windows.1
[application/octet-stream] HEAD_v14-0002-Add-clarification-for-the-behaviour-of-the-publi.patch (2.4K, ../../OS3PR01MB6275FB5397C6A647F262A3A69E009@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-HEAD_v14-0002-Add-clarification-for-the-behaviour-of-the-publi.patch)
download | inline diff:
From 6917fc349c36fe153b385bf53dfc8b2ed9c3754e Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 28 Sep 2022 14:35:41 +0800
Subject: [PATCH v14 2/2] Add clarification for the behaviour of the
publication parameter publish_via_partition_root
Assume a subscription is subscribing to multiple publications, and these
publications publish a partitioned table and its partitions respectively:
[publisher-side]
create table parent (a int primary key) partition by range (a);
create table child partition of parent default;
create publication pub1 for table parent;
create publication pub2 for table child;
[subscriber-side]
create subscription sub connection 'xxxx' publication pub1, pub2;
The manual does not clearly describe the behaviour when the user had
specified the parameter 'publish_via_partition_root' on just one of
the publications. This patch modifies documentation to clarify the
following rules:
- If the parameter publish_via_partition_root is specified only in pub1,
changes will be published using the identity and schema of the table 'parent'.
- If the parameter publish_via_partition_root is specified only in pub2,
changes will be published using the identity and schema of the table 'child'.
---
doc/src/sgml/ref/create_publication.sgml | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml
index e229384e6f..74d50dd9fb 100644
--- a/doc/src/sgml/ref/create_publication.sgml
+++ b/doc/src/sgml/ref/create_publication.sgml
@@ -201,6 +201,16 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
consisting of a different set of partitions.
</para>
+ <para>
+ There can be a case where a subscription combines multiple
+ publications. If a root partitioned table is published by any
+ subscribed publications which set
+ <literal>publish_via_partition_root</literal> = true, changes on this
+ root partitioned table (or on its partitions) will be published using
+ the identity and schema of this root partitioned table rather than
+ that of the individual partitions.
+ </para>
+
<para>
This parameter also affects how row filters and column lists are
chosen for partitions; see below for details.
--
2.23.0.windows.1
[application/octet-stream] REL15_v14-0001-Fix-data-replicated-twice-when-specifying-publis_patch (8.6K, ../../OS3PR01MB6275FB5397C6A647F262A3A69E009@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL15_v14-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL14_v14-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275FB5397C6A647F262A3A69E009@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL14_v14-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-11 05:44 [email protected] <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-11-11 05:44 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Fri, Oct 21, 2022 at 17:02 PM Peter Smith <[email protected]> wrote:
> Here are my review comments for HEAD patches v13*
Thanks for your comments.
> Patch HEAD_v13-0002
>
> 1. Commit message
>
> The following usage scenarios are not described in detail in the manual:
> If one subscription subscribes multiple publications, and these publications
> publish a partitioned table and its partitions respectively. When we specify
> this parameter on one or more of these publications, which identity and schema
> should be used to publish the changes?
>
> In these cases, I think the parameter publish_via_partition_root behave as
> follows:
>
> ~
>
> It seemed worded a bit strangely. Also, you said "on one or more of
> these publications" but the examples only show only one publication
> having 'publish_via_partition_root'.
>
> SUGGESTION (I've modified the wording slightly but the examples are
> unchanged).
>
> Assume a subscription is subscribing to multiple publications, and
> these publications publish a partitioned table and its partitions
> respectively:
>
> [publisher-side]
> create table parent (a int primary key) partition by range (a);
> create table child partition of parent default;
>
> create publication pub1 for table parent;
> create publication pub2 for table child;
>
> [subscriber-side]
> create subscription sub connection 'xxxx' publication pub1, pub2;
>
> The manual does not clearly describe the behaviour when the user had
> specified the parameter 'publish_via_partition_root' on just one of
> the publications. This patch modifies documentation to clarify the
> following rules:
>
> - If the parameter publish_via_partition_root is specified only in pub1,
> changes will be published using the identity and schema of the table 'parent'.
>
> - If the parameter publish_via_partition_root is specified only in pub2,
> changes will be published using the identity and schema of the table 'child'.
Improved as suggested.
> ~~~
>
> 2.
>
> - If the parameter publish_via_partition_root is specified only in pub2,
> changes will be published using the identity and schema of the table child.
>
> ~
>
> Is that right though? This rule seems 100% contrary to the meaning of
> 'publish_via_partition_root=true'.
Yes, I think this behaviour fits the meaning of publish_via_partition_root.
Please refer to this description in the document:
```
This parameter determines whether changes in a partitioned table (or on its
partitions) contained in the publication will be published ...
```
So I think for 'publish_via_partition_root' to work, the partitioned table must
be specified in this publication.
Since only one member (partition table 'child') of this partition tree
('parent', 'child') is specified in 'pub2', even if 'pub2' specifies the
parameter 'publish_via_partition_root', 'pub2' will publish changes using the
identity and schema of the table 'child'.
> ------
>
> 3. doc/src/sgml/ref/create_publication.sgml
>
> + <para>
> + If a root partitioned table is published by any subscribed
> publications which
> + set publish_via_partition_root = true, changes on this root
> partitioned table
> + (or on its partitions) will be published using the identity
> and schema of this
> + root partitioned table rather than that of the individual partitions.
> + </para>
>
> This seems to only describe the first example from the commit message.
> What about some description to explain the second example?
I think the second example is already described in the pg-doc (please see the
reply to #2). I am not quite sure if additional modifications are required. Do
you have any suggestions?
The new patch set was attached in [1].
[1] - https://www.postgresql.org/message-id/OS3PR01MB6275FB5397C6A647F262A3A69E009%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-11 05:45 [email protected] <[email protected]>
parent: Takamichi Osumi (Fujitsu) <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-11-11 05:45 UTC (permalink / raw)
To: Takamichi Osumi (Fujitsu) <[email protected]>; Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>; vignesh C <[email protected]>
On Tues, Nov 8, 2022 at 12:12 PM Osumi, Takamichi/大墨 昂道 <[email protected]> wrote:
> On Monday, October 17, 2022 2:49 PM Wang, Wei/王 威
> <[email protected]> wrote:
> > Attach the new patch set.
> Hi, thank you for posting the new patches.
>
>
> Here are minor comments on the HEAD_v13-0002.
Thanks for your comments.
> (1) Suggestion for the document description
>
> + <para>
> + If a root partitioned table is published by any subscribed publications
> which
> + set publish_via_partition_root = true, changes on this root partitioned
> table
> + (or on its partitions) will be published using the identity and schema of this
> + root partitioned table rather than that of the individual partitions.
> + </para>
> +
>
> I suppose this sentence looks quite similar to the one in the previous paragraph
> and can be adjusted.
>
> IIUC the main value of the patch is to clarify what happens when
> we mix publications of different publish_via_partition_root settings for one
> partition hierarchy.
> If this is true, how about below sentence instead of the one above ?
>
> "
> There can be a case where a subscription combines publications with
> different publish_via_partition_root values for one same partition hierarchy
> (e.g. subscribe two publications indicating the root partitioned table and its child
> table respectively).
> In this case, the identity and schema of the root partitioned table take priority.
> "
Thanks for your suggestion.
I agree that we should mention that this description is for a case where one
subscription subscribes to multiple publications. And I think it would be
better if we mentioned that the option publish_via_partition_root is specified
on a publication that publishes a root partitioned table. So I added the
description of this case as you suggested.
> (2) Better documentation alignment
>
> I think we need to wrap publish_via_partition_root by "literal" tag
> in the documentation create_publication.sgml.
Improved.
The new patch set was attached in [1].
[1] - https://www.postgresql.org/message-id/OS3PR01MB6275FB5397C6A647F262A3A69E009%40OS3PR01MB6275.jpnprd0...
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-13 16:55 vignesh C <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: vignesh C @ 2022-11-13 16:55 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Fri, 11 Nov 2022 at 11:13, [email protected]
<[email protected]> wrote:
>
> On Fri, Oct 21, 2022 at 17:02 PM Peter Smith <[email protected]> wrote:
> >
>
> Thanks for your comments. Sorry for not replying in time.
>
> > On Mon, Oct 17, 2022 at 4:49 PM [email protected]
> > <[email protected]> wrote:
> > >
> > > On Wed, Oct 5, 2022 at 11:08 AM Peter Smith <[email protected]>
> > wrote:
> > > > Hi Wang-san. Here are my review comments for HEAD_v12-0001 patch.
> > >
> > ...
> > > >
> > > > 3. QUESTION - pg_get_publication_tables / fetch_table_list
> > > >
> > > > When the same table is published by different publications (but there
> > > > are other differences like row-filters/column-lists in each
> > > > publication) the result tuple of this function does not include the
> > > > pubid. Maybe the SQL of pg_publication_tables/fetch_table_list() is OK
> > > > as-is but how does it manage to associate each table with the correct
> > > > tuple?
> > > >
> > > > I know it apparently all seems to work but I’m not how does that
> > > > happen? Can you explain why a puboid is not needed for the result
> > > > tuple of this function?
> > >
> > > Sorry, I am not sure I understand your question.
> > > I try to answer your question by explaining the two functions you mentioned:
> > >
> > > First, the function pg_get_publication_tables gets the list (see table_infos)
> > > that included published table and the corresponding publication. Then based
> > > on this list, the function pg_get_publication_tables returns information
> > > (scheme, relname, row filter and column list) about the published tables in the
> > > publications list. It just doesn't return pubid.
> > >
> > > Then, the SQL in the function fetch_table_list will get the columns in the
> > > column list from pg_attribute. (This is to return all columns when the column
> > > list is not specified)
> > >
> >
> > I meant, for example, if the different publications specified
> > different col-lists for the same table then IIUC the
> > fetch_table_lists() is going to return 2 list elements
> > (schema,rel_name,row_filter,col_list). But when the schema/rel_name
> > are the same for 2 elements then (without also a pubid) how are you
> > going to know where the list element came from, and how come that is
> > not important?
> >
> > > > ~~
> > > >
> > > > test_pub=# create table t1(a int, b int, c int);
> > > > CREATE TABLE
> > > > test_pub=# create publication pub1 for table t1(a) where (a > 99);
> > > > CREATE PUBLICATION
> > > > test_pub=# create publication pub2 for table t1(a,b) where (b < 33);
> > > > CREATE PUBLICATION
> > > >
> > > > Following seems OK when I swap orders of publication names...
> > > >
> > > > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > > > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > > > ARRAY['pub2','pub1']) gpt(relid, attrs, qual);
> > > > relid | attrs | rowfilter
> > > > -------+-------+-----------
> > > > 16385 | 1 2 | (b < 33)
> > > > 16385 | 1 | (a > 99)
> > > > (2 rows)
> > > >
> > > > test_pub=# SELECT gpt.relid, gpt.attrs, pg_get_expr(gpt.qual,
> > > > gpt.relid) AS rowfilter from pg_get_publication_tables(VARIADIC
> > > > ARRAY['pub1','pub2']) gpt(relid, attrs, qual);
> > > > relid | attrs | rowfilter
> > > > -------+-------+-----------
> > > > 16385 | 1 | (a > 99)
> > > > 16385 | 1 2 | (b < 33)
> > > > (2 rows)
> > > >
> > > > But what about this (this is similar to the SQL fragment from
> > > > fetch_table_list); I swapped the pub names but the results are the
> > > > same...
> > > >
> > > > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > > > array_agg(p.pubname)) from pg_publication p where pubname
> > > > IN('pub2','pub1');
> > > >
> > > > pg_get_publication_tables
> > > >
> > > > ---------------------------------------------------------------------------------------------
> > ----
> > > > ---------------------------------------------------------------------
> > > > ---------------------------------------------------------------------------------------------
> > ----
> > > > ---------------------------------------------------------------------
> > > > -------------------------------------------------------------------
> > > > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > > > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > > > :vartype 23 :vartypmod -1 :var
> > > > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > > :constbyval true :constisnull false :
> > > > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > > > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > > > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > > > :varattno 2 :vartype 23 :vartypmod -1 :v
> > > > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > > :constbyval true :constisnull false
> > > > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > > > (2 rows)
> > > >
> > > > test_pub=# SELECT pg_get_publication_tables(VARIADIC
> > > > array_agg(p.pubname)) from pg_publication p where pubname
> > > > IN('pub1','pub2');
> > > >
> > > > pg_get_publication_tables
> > > >
> > > > ---------------------------------------------------------------------------------------------
> > ----
> > > > ---------------------------------------------------------------------
> > > > ---------------------------------------------------------------------------------------------
> > ----
> > > > ---------------------------------------------------------------------
> > > > -------------------------------------------------------------------
> > > > (16385,1,"{OPEXPR :opno 521 :opfuncid 147 :opresulttype 16 :opretset
> > > > false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1 :varattno 1
> > > > :vartype 23 :vartypmod -1 :var
> > > > collid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location 47}
> > > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > > :constbyval true :constisnull false :
> > > > location 51 :constvalue 4 [ 99 0 0 0 0 0 0 0 ]}) :location 49}")
> > > > (16385,"1 2","{OPEXPR :opno 97 :opfuncid 66 :opresulttype 16
> > > > :opretset false :opcollid 0 :inputcollid 0 :args ({VAR :varno 1
> > > > :varattno 2 :vartype 23 :vartypmod -1 :v
> > > > arcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location 49}
> > > > {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4
> > > > :constbyval true :constisnull false
> > > > :location 53 :constvalue 4 [ 33 0 0 0 0 0 0 0 ]}) :location 51}")
> > > > (2 rows)
> > >
> > > I think this is because the usage of SELECT statement. The order seems
> > depend
> > > on pg_publication. Such as:
> > >
> > > postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE
> > pubname IN ('pub1','pub2');
> > > array_agg
> > > -------------
> > > {pub1,pub2}
> > > (1 row)
> > >
> > > postgres=# SELECT array_agg(p.pubname) FROM pg_publication p WHERE
> > pubname IN ('pub2','pub1');
> > > array_agg
> > > -------------
> > > {pub1,pub2}
> > > (1 row)
> > >
> >
> > Right, so I felt it was a bit dubious that the result of the function
> > “seems to depend on” something. That’s why I was asking why the list
> > elements did not include a pubid. Then a caller could be certain what
> > element belonged with what publication. It's not quite clear to me why
> > that is not important for this patch - but anyway, even if it's not
> > necessary for this patch's usage, this is a function that is exposed
> > to users who might have different needs/expectations than this patch
> > has, so shouldn't the result be less fuzzy for them?
>
> Yes, I agree that there may be such a need in the future. Added 'pubid' to the
> output of this function.
> BTW, I think the usage of the function pg_get_publication_tables is not
> documented in the pg-doc now, it doesn't seem to be a function provided to
> users. So I didn't modify the documentation.
>
> Attach new patches.
Here we are having tables list to store the relids and table_infos
list which stores pubid along with relid. Here tables list acts as a
temporary list to get filter_partitions and then delete the
published_rel from table_infos. Will it be possible to directly
operate on table_infos list and remove the temporary tables list used.
We might have to implement comparator, deduplication functions and
change filter_partitions function to work directly on published_rel
type list.
+ /
+ * Record the published table and the
corresponding publication so
+ * that we can get row filters and column list later.
+ *
+ * When a table is published by multiple
publications, to obtain
+ * all row filters and column list, the
structure related to this
+ * table will be recorded multiple times.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info =
(published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
+
+ tables = list_concat(tables, pub_elem_tables);
Thoughts?
Regards,
Vignesh
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-16 08:58 [email protected] <[email protected]>
parent: vignesh C <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: [email protected] @ 2022-11-16 08:58 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: Peter Smith <[email protected]>; Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Mon, Nov 14, 2022 at 0:56 AM vignesh C <[email protected]> wrote:
> >
> > Attach new patches.
>
Thanks for your comments.
> Here we are having tables list to store the relids and table_infos
> list which stores pubid along with relid. Here tables list acts as a
> temporary list to get filter_partitions and then delete the
> published_rel from table_infos. Will it be possible to directly
> operate on table_infos list and remove the temporary tables list used.
> We might have to implement comparator, deduplication functions and
> change filter_partitions function to work directly on published_rel
> type list.
> + /
> + * Record the published table and the
> corresponding publication so
> + * that we can get row filters and column list later.
> + *
> + * When a table is published by multiple
> publications, to obtain
> + * all row filters and column list, the
> structure related to this
> + * table will be recorded multiple times.
> + */
> + foreach(lc, pub_elem_tables)
> + {
> + published_rel *table_info =
> (published_rel *) malloc(sizeof(published_rel));
> +
> + table_info->relid = lfirst_oid(lc);
> + table_info->pubid = pub_elem->oid;
> + table_infos = lappend(table_infos, table_info);
> + }
> +
> + tables = list_concat(tables, pub_elem_tables);
>
> Thoughts?
I think we could only deduplicate published tables per publication to get all
row filters and column lists for each published table later.
I removed the temporary list 'tables' and modified the API of the function
filter_partitions to handle published_rel type list.
Attach the new patch set.
Regards,
Wang wei
Attachments:
[application/octet-stream] HEAD_v15-0001-Fix-data-replicated-twice-when-specifying-publis.patch (22.5K, ../../OS3PR01MB6275C62FF1D1CDE4FCBF61339E079@OS3PR01MB6275.jpnprd01.prod.outlook.com/2-HEAD_v15-0001-Fix-data-replicated-twice-when-specifying-publis.patch)
download | inline diff:
From 3563e90bbb5b9718eb7a233378b017692945d2bf Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Tue, 14 Jun 2022 13:44:09 +0800
Subject: [PATCH v15 1/2] Fix data replicated twice when specifying
publish_via_partition_root option.
If there are two publications - one of them publishing a parent table
(using publish_via_partition_root = true) and the other is publishing one of
the parent's child tables - then subscribing to both publications from one
subscription results in the same initial child data being copied twice. It
should only be copied once.
To fix this, the API function pg_get_publication_tables has been extended to
take a publication list. Now, when getting the table information, if the
publish_via_partition_root is true, the function can exclude a partition table
whose ancestor is also published by the same publication list.
---
src/backend/catalog/pg_publication.c | 200 ++++++++++++++-------
src/backend/commands/subscriptioncmds.c | 81 +++++++--
src/include/catalog/pg_proc.dat | 12 +-
src/test/regress/expected/rules.out | 2 +-
src/test/subscription/t/013_partition.pl | 18 +-
src/test/subscription/t/028_row_filter.pl | 13 +-
src/test/subscription/t/031_column_list.pl | 5 +-
src/tools/pgindent/typedefs.list | 1 +
8 files changed, 228 insertions(+), 104 deletions(-)
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 59967098b3..e6a0acb7bf 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -45,6 +45,14 @@
#include "utils/rel.h"
#include "utils/syscache.h"
+/* Records association between publication and published table */
+typedef struct
+{
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication that publishes this
+ * table. */
+} published_rel;
+
static void publication_translate_columns(Relation targetrel, List *columns,
int *natts, AttrNumber **attrs);
@@ -175,39 +183,45 @@ pg_relation_is_publishable(PG_FUNCTION_ARGS)
* Filter out the partitions whose parent tables were also specified in
* the publication.
*/
-static List *
-filter_partitions(List *relids)
+static void
+filter_partitions(List *table_infos)
{
- List *result = NIL;
ListCell *lc;
- ListCell *lc2;
- foreach(lc, relids)
+ foreach(lc, table_infos)
{
- bool skip = false;
- List *ancestors = NIL;
- Oid relid = lfirst_oid(lc);
+ bool skip = false;
+ List *ancestors = NIL;
+ ListCell *lc2;
+ published_rel *table_info = (published_rel *) lfirst(lc);
- if (get_rel_relispartition(relid))
- ancestors = get_partition_ancestors(relid);
+ if (get_rel_relispartition(table_info->relid))
+ ancestors = get_partition_ancestors(table_info->relid);
foreach(lc2, ancestors)
{
Oid ancestor = lfirst_oid(lc2);
+ ListCell *lc3;
/* Check if the parent table exists in the published table list. */
- if (list_member_oid(relids, ancestor))
+ foreach(lc3, table_infos)
{
- skip = true;
- break;
+ Oid relid = ((published_rel *) lfirst(lc3))->relid;
+
+ if (relid == ancestor)
+ {
+ skip = true;
+ break;
+ }
}
+
+ if (skip)
+ break;
}
- if (!skip)
- result = lappend_oid(result, relid);
+ if (skip)
+ table_infos = foreach_delete_current(table_infos, lc);
}
-
- return result;
}
/*
@@ -1026,22 +1040,28 @@ GetPublicationByName(const char *pubname, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Get information of the tables in the given publication array.
+ *
+ * Returns pubid, relid, column list, row filter for each table.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
-#define NUM_PUBLICATION_TABLES_ELEM 3
+#define NUM_PUBLICATION_TABLES_ELEM 4
FuncCallContext *funcctx;
- char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
- Publication *publication;
- List *tables;
+ List *table_infos = NIL;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
MemoryContext oldcontext;
+ ArrayType *arr;
+ Datum *elems;
+ int nelems,
+ i;
+ bool viaroot = false;
+ ListCell *lc;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
@@ -1049,68 +1069,109 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- publication = GetPublicationByName(pubname, false);
-
/*
- * Publications support partitioned tables, although all changes are
- * replicated using leaf partition identity and schema, so we only
- * need those.
+ * Deconstruct the parameter into elements where each element is a
+ * publication name.
*/
- if (publication->alltables)
- {
- tables = GetAllTablesPublicationRelations(publication->pubviaroot);
- }
- else
+ arr = PG_GETARG_ARRAYTYPE_P(0);
+ deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT,
+ &elems, NULL, &nelems);
+
+ /* Get Oids of tables from each publication. */
+ for (i = 0; i < nelems; i++)
{
- List *relids,
- *schemarelids;
-
- relids = GetPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- schemarelids = GetAllSchemaPublicationRelations(publication->oid,
- publication->pubviaroot ?
- PUBLICATION_PART_ROOT :
- PUBLICATION_PART_LEAF);
- tables = list_concat_unique_oid(relids, schemarelids);
+ Publication *pub_elem;
+ List *pub_elem_tables = NIL;
+
+ pub_elem = GetPublicationByName(TextDatumGetCString(elems[i]), false);
/*
- * If the publication publishes partition changes via their
- * respective root partitioned tables, we must exclude partitions
- * in favor of including the root partitioned tables. Otherwise,
- * the function could return both the child and parent tables
- * which could cause data of the child table to be
- * double-published on the subscriber side.
+ * Publications support partitioned tables. If
+ * publish_via_partition_root is false, all changes are replicated
+ * using leaf partition identity and schema, so we only need those.
+ * Otherwise, get the partitioned table itself.
*/
- if (publication->pubviaroot)
- tables = filter_partitions(tables);
+ if (pub_elem->alltables)
+ pub_elem_tables = GetAllTablesPublicationRelations(pub_elem->pubviaroot);
+ else
+ {
+ List *relids,
+ *schemarelids;
+
+ relids = GetPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ schemarelids = GetAllSchemaPublicationRelations(pub_elem->oid,
+ pub_elem->pubviaroot ?
+ PUBLICATION_PART_ROOT :
+ PUBLICATION_PART_LEAF);
+ pub_elem_tables = list_concat_unique_oid(relids, schemarelids);
+ }
+
+ /*
+ * Record the published table and the corresponding publication so
+ * that we can get row filters and column list later.
+ *
+ * When a table is published by multiple publications, to obtain
+ * all row filters and column list, the structure related to this
+ * table will be recorded multiple times.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info = (published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
+
+ /* At least one publication is using publish_via_partition_root. */
+ if (pub_elem->pubviaroot)
+ viaroot = true;
}
+ pfree(elems);
+
+ /*
+ * If the publication publishes partition changes via their respective
+ * root partitioned tables, we must exclude partitions in favor of
+ * including the root partitioned tables. Otherwise, the function
+ * could return both the child and parent tables which could cause
+ * data of the child table to be double-published on the subscriber
+ * side.
+ */
+ if (viaroot)
+ filter_partitions(table_infos);
+
/* Construct a tuple descriptor for the result rows. */
tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATION_TABLES_ELEM);
- TupleDescInitEntry(tupdesc, (AttrNumber) 1, "relid",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pubid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "relid",
OIDOID, -1, 0);
- TupleDescInitEntry(tupdesc, (AttrNumber) 2, "attrs",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "attrs",
INT2VECTOROID, -1, 0);
- TupleDescInitEntry(tupdesc, (AttrNumber) 3, "qual",
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "qual",
PG_NODE_TREEOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
- funcctx->user_fctx = (void *) tables;
+ funcctx->user_fctx = (void *) table_infos;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- tables = (List *) funcctx->user_fctx;
+ table_infos = (List *) funcctx->user_fctx;
- if (funcctx->call_cntr < list_length(tables))
+ if (funcctx->call_cntr < list_length(table_infos))
{
HeapTuple pubtuple = NULL;
HeapTuple rettuple;
- Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Publication *pub;
+ published_rel *table_info = (published_rel *) list_nth(table_infos, funcctx->call_cntr);
+ Oid relid = table_info->relid;
Oid schemaid = get_rel_namespace(relid);
Datum values[NUM_PUBLICATION_TABLES_ELEM] = {0};
bool nulls[NUM_PUBLICATION_TABLES_ELEM] = {0};
@@ -1119,38 +1180,39 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
* Form tuple with appropriate data.
*/
- publication = GetPublicationByName(pubname, false);
+ pub = GetPublication(table_info->pubid);
- values[0] = ObjectIdGetDatum(relid);
+ values[0] = ObjectIdGetDatum(pub->oid);
+ values[1] = ObjectIdGetDatum(relid);
/*
* We don't consider row filters or column lists for FOR ALL TABLES or
* FOR TABLES IN SCHEMA publications.
*/
- if (!publication->alltables &&
+ if (!pub->alltables &&
!SearchSysCacheExists2(PUBLICATIONNAMESPACEMAP,
ObjectIdGetDatum(schemaid),
- ObjectIdGetDatum(publication->oid)))
+ ObjectIdGetDatum(pub->oid)))
pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
ObjectIdGetDatum(relid),
- ObjectIdGetDatum(publication->oid));
+ ObjectIdGetDatum(pub->oid));
if (HeapTupleIsValid(pubtuple))
{
/* Lookup the column list attribute. */
- values[1] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
Anum_pg_publication_rel_prattrs,
- &(nulls[1]));
+ &(nulls[2]));
/* Null indicates no filter. */
- values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ values[3] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
Anum_pg_publication_rel_prqual,
- &(nulls[2]));
+ &(nulls[3]));
}
else
{
- nulls[1] = true;
nulls[2] = true;
+ nulls[3] = true;
}
rettuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d673557ea4..804ad912b0 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1920,25 +1920,84 @@ static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
{
WalRcvExecResult *res;
- StringInfoData cmd;
+ StringInfoData cmd,
+ pub_names;
TupleTableSlot *slot;
Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
- bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
+ int server_version = walrcv_server_version(wrconn);
+ bool check_columnlist = (server_version >= 150000);
- initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+ initStringInfo(&pub_names);
+ get_publications_str(publications, &pub_names, true);
- /* Get column lists for each relation if the publisher supports it */
- if (check_columnlist)
- appendStringInfoString(&cmd, ", t.attnames\n");
+ initStringInfo(&cmd);
- appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
- " WHERE t.pubname IN (");
- get_publications_str(publications, &cmd, true);
- appendStringInfoChar(&cmd, ')');
+ /*
+ * Get namespace, relname and column list (if supported) of the tables
+ * belonging to the specified publications.
+ *
+ * Get the list of tables from the publisher. The partition table whose
+ * ancestor is also in this list will be ignored, otherwise the initial
+ * data in the partition table would be replicated twice.
+ *
+ * From version 16, the parameter of the function
+ * pg_get_publication_tables can be an array of publications. The
+ * partition table whose ancestor is also published in this publication
+ * array will be filtered out in this function.
+ */
+ if (server_version >= 160000)
+ {
+ appendStringInfo(&cmd, "SELECT DISTINCT N.nspname, C.relname,\n"
+ " ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n"
+ " FROM pg_class C\n"
+ " JOIN pg_namespace N ON N.oid = C.relnamespace\n"
+ " JOIN ( SELECT (pg_get_publication_tables(VARIADIC array_agg(pubname::text))).*\n"
+ " FROM pg_publication\n"
+ " WHERE pubname IN ( %s )) as GPT\n"
+ " ON GPT.relid = C.oid\n",
+ pub_names.data);
+ }
+ else
+ {
+ appendStringInfoString(&cmd, "WITH pub_tabs AS(\n"
+ " SELECT DISTINCT N.nspname, C.oid, C.relname, C.relispartition\n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", ( SELECT array_agg(a.attname ORDER BY a.attnum)\n"
+ " FROM pg_attribute a\n"
+ " WHERE a.attrelid = GPT.relid AND a.attnum > 0 AND\n"
+ " NOT a.attisdropped AND\n"
+ " (a.attnum = ANY(GPT.attrs) OR GPT.attrs IS NULL)\n"
+ " ) AS attnames\n");
+
+ appendStringInfo(&cmd, " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid AND P.pubname IN ( %s )\n"
+ ")\n"
+ "SELECT DISTINCT pub_tabs.nspname, pub_tabs.relname\n",
+ pub_names.data);
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", pub_tabs.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pub_tabs\n"
+ " WHERE (pub_tabs.relispartition IS FALSE\n"
+ " OR NOT EXISTS (SELECT 1 FROM pg_partition_ancestors(pub_tabs.oid) as PA\n"
+ " WHERE PA.relid IN (SELECT pub_tabs.oid FROM pub_tabs)\n"
+ " AND PA.relid != pub_tabs.oid))\n");
+ }
res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
+ pfree(pub_names.data);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 9dbe9ec801..8344ff1b57 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11615,11 +11615,13 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
- proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'record', proargtypes => 'text',
- proallargtypes => '{text,oid,int2vector,pg_node_tree}',
- proargmodes => '{i,o,o,o}', proargnames => '{pubname,relid,attrs,qual}',
+{ oid => '6119',
+ descr => 'get information of the tables in the given publication array',
+ proname => 'pg_get_publication_tables', prorows => '1000',
+ provariadic => 'text', proretset => 't', provolatile => 's',
+ prorettype => 'record', proargtypes => '_text',
+ proallargtypes => '{_text,oid,oid,int2vector,pg_node_tree}',
+ proargmodes => '{v,o,o,o,o}', proargnames => '{pubname,pubid,relid,attrs,qual}',
prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..5d595db68b 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1446,7 +1446,7 @@ pg_publication_tables| SELECT p.pubname,
WHERE ((a.attrelid = gpt.relid) AND (a.attnum > 0) AND (NOT a.attisdropped) AND ((a.attnum = ANY ((gpt.attrs)::smallint[])) OR (gpt.attrs IS NULL)))) AS attnames,
pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
+ LATERAL pg_get_publication_tables(VARIADIC ARRAY[(p.pubname)::text]) gpt(pubid, relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl
index 8b33e4e7ae..100e2d2fd0 100644
--- a/src/test/subscription/t/013_partition.pl
+++ b/src/test/subscription/t/013_partition.pl
@@ -473,12 +473,12 @@ $node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab3_1 (a int PRIMARY KEY, c text DEFAULT 'sub2_tab3_1', b text)"
);
-# Note: We create two separate tables, not a partitioned one, so that we can
-# easily identity through which relation were the changes replicated.
+# Note: We only create one table (tab4) here. We specified
+# publish_via_partition_root = true (see pub_all and pub_lower_level above), so
+# all data will be replicated to that table.
$node_subscriber2->safe_psql('postgres',
"CREATE TABLE tab4 (a int PRIMARY KEY)");
-$node_subscriber2->safe_psql('postgres',
- "CREATE TABLE tab4_1 (a int PRIMARY KEY)");
+
# Publication that sub2 points to now publishes via root, so must update
# subscription target relations. We set the list of publications so that
# the FOR ALL TABLES publication is second (the list order matters).
@@ -550,11 +550,6 @@ $result =
$node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4 ORDER BY 1");
is($result, qq(0), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# now switch the order of publications in the list, try again, the result
# should be the same (no dependence on order of pulications)
$node_subscriber2->safe_psql('postgres',
@@ -577,11 +572,6 @@ $result =
is( $result, qq(0
1), 'inserts into tab4 replicated');
-$result =
- $node_subscriber2->safe_psql('postgres', "SELECT a FROM tab4_1 ORDER BY 1");
-is($result, qq(), 'inserts into tab4_1 replicated');
-
-
# update (replicated as update)
$node_publisher->safe_psql('postgres', "UPDATE tab1 SET a = 6 WHERE a = 5");
$node_publisher->safe_psql('postgres', "UPDATE tab2 SET a = 6 WHERE a = 5");
diff --git a/src/test/subscription/t/028_row_filter.pl b/src/test/subscription/t/028_row_filter.pl
index da52289dde..056ceb8096 100644
--- a/src/test/subscription/t/028_row_filter.pl
+++ b/src/test/subscription/t/028_row_filter.pl
@@ -386,6 +386,10 @@ $node_publisher->safe_psql('postgres',
"INSERT INTO tab_rowfilter_child(a, b) VALUES(0,'0'),(30,'30'),(40,'40')"
);
+# insert data into partitioned table.
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab_rowfilter_viaroot_part(a) VALUES(13), (17)");
+
$node_subscriber->safe_psql('postgres',
"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub_1, tap_pub_2, tap_pub_3, tap_pub_4a, tap_pub_4b, tap_pub_5a, tap_pub_5b, tap_pub_toast, tap_pub_inherits, tap_pub_viaroot_2, tap_pub_viaroot_1"
);
@@ -707,13 +711,18 @@ is($result, qq(t|1), 'check replicated rows to tab_rowfilter_toast');
# the row filter for the top-level ancestor:
#
# tab_rowfilter_viaroot_part filter is: (a > 15)
+# - INSERT (13) NO, 13 < 15
# - INSERT (14) NO, 14 < 15
# - INSERT (15) NO, 15 = 15
# - INSERT (16) YES, 16 > 15
+# - INSERT (17) YES, 17 > 15
$result =
$node_subscriber->safe_psql('postgres',
- "SELECT a FROM tab_rowfilter_viaroot_part");
-is($result, qq(16), 'check replicated rows to tab_rowfilter_viaroot_part');
+ "SELECT a FROM tab_rowfilter_viaroot_part ORDER BY 1");
+is($result, qq(16
+17),
+ 'check replicated rows to tab_rowfilter_viaroot_part'
+);
# Check there is no data in tab_rowfilter_viaroot_part_1 because rows are
# replicated via the top most parent table tab_rowfilter_viaroot_part
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index ae022faa78..a39f3d27ff 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -959,7 +959,8 @@ $node_publisher->safe_psql(
CREATE TABLE test_root_1 PARTITION OF test_root FOR VALUES FROM (1) TO (10);
CREATE TABLE test_root_2 PARTITION OF test_root FOR VALUES FROM (10) TO (20);
- CREATE PUBLICATION pub_root_true FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_1 FOR TABLE test_root (a) WITH (publish_via_partition_root = true);
+ CREATE PUBLICATION pub_root_true_2 FOR TABLE test_root_1 (a, b) WITH (publish_via_partition_root = true);
-- initial data
INSERT INTO test_root VALUES (1, 2, 3);
@@ -968,7 +969,7 @@ $node_publisher->safe_psql(
$node_subscriber->safe_psql(
'postgres', qq(
- CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true;
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_root_true_1, pub_root_true_2;
));
$node_subscriber->wait_for_subscription_sync;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 245aea1dd1..d64b0f6390 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3581,6 +3581,7 @@ pthread_mutex_t
pthread_once_t
pthread_t
ptrdiff_t
+published_rel
pull_var_clause_context
pull_varattnos_context
pull_varnos_context
--
2.23.0.windows.1
[application/octet-stream] HEAD_v15-0002-Add-clarification-for-the-behaviour-of-the-publi.patch (2.4K, ../../OS3PR01MB6275C62FF1D1CDE4FCBF61339E079@OS3PR01MB6275.jpnprd01.prod.outlook.com/3-HEAD_v15-0002-Add-clarification-for-the-behaviour-of-the-publi.patch)
download | inline diff:
From eb313d87b9015a32efe093dad272ff48bfdc08df Mon Sep 17 00:00:00 2001
From: wangw <[email protected]>
Date: Wed, 28 Sep 2022 14:35:41 +0800
Subject: [PATCH v15 2/2] Add clarification for the behaviour of the
publication parameter publish_via_partition_root
Assume a subscription is subscribing to multiple publications, and these
publications publish a partitioned table and its partitions respectively:
[publisher-side]
create table parent (a int primary key) partition by range (a);
create table child partition of parent default;
create publication pub1 for table parent;
create publication pub2 for table child;
[subscriber-side]
create subscription sub connection 'xxxx' publication pub1, pub2;
The manual does not clearly describe the behaviour when the user had
specified the parameter 'publish_via_partition_root' on just one of
the publications. This patch modifies documentation to clarify the
following rules:
- If the parameter publish_via_partition_root is specified only in pub1,
changes will be published using the identity and schema of the table 'parent'.
- If the parameter publish_via_partition_root is specified only in pub2,
changes will be published using the identity and schema of the table 'child'.
---
doc/src/sgml/ref/create_publication.sgml | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml
index e229384e6f..74d50dd9fb 100644
--- a/doc/src/sgml/ref/create_publication.sgml
+++ b/doc/src/sgml/ref/create_publication.sgml
@@ -201,6 +201,16 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
consisting of a different set of partitions.
</para>
+ <para>
+ There can be a case where a subscription combines multiple
+ publications. If a root partitioned table is published by any
+ subscribed publications which set
+ <literal>publish_via_partition_root</literal> = true, changes on this
+ root partitioned table (or on its partitions) will be published using
+ the identity and schema of this root partitioned table rather than
+ that of the individual partitions.
+ </para>
+
<para>
This parameter also affects how row filters and column lists are
chosen for partitions; see below for details.
--
2.23.0.windows.1
[application/octet-stream] REL14_v15-0001-Fix-data-replicated-twice-when-specifying-publis_patch (5.3K, ../../OS3PR01MB6275C62FF1D1CDE4FCBF61339E079@OS3PR01MB6275.jpnprd01.prod.outlook.com/4-REL14_v15-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
[application/octet-stream] REL15_v15-0001-Fix-data-replicated-twice-when-specifying-publis_patch (8.6K, ../../OS3PR01MB6275C62FF1D1CDE4FCBF61339E079@OS3PR01MB6275.jpnprd01.prod.outlook.com/5-REL15_v15-0001-Fix-data-replicated-twice-when-specifying-publis_patch)
download
^ permalink raw reply [nested|flat] 99+ messages in thread
* Re: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-17 05:57 vignesh C <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 99+ messages in thread
From: vignesh C @ 2022-11-17 05:57 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Peter Smith <[email protected]>; Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Wed, 16 Nov 2022 at 14:28, [email protected]
<[email protected]> wrote:
>
> On Mon, Nov 14, 2022 at 0:56 AM vignesh C <[email protected]> wrote:
> > >
> > > Attach new patches.
> >
>
> Thanks for your comments.
>
> > Here we are having tables list to store the relids and table_infos
> > list which stores pubid along with relid. Here tables list acts as a
> > temporary list to get filter_partitions and then delete the
> > published_rel from table_infos. Will it be possible to directly
> > operate on table_infos list and remove the temporary tables list used.
> > We might have to implement comparator, deduplication functions and
> > change filter_partitions function to work directly on published_rel
> > type list.
> > + /
> > + * Record the published table and the
> > corresponding publication so
> > + * that we can get row filters and column list later.
> > + *
> > + * When a table is published by multiple
> > publications, to obtain
> > + * all row filters and column list, the
> > structure related to this
> > + * table will be recorded multiple times.
> > + */
> > + foreach(lc, pub_elem_tables)
> > + {
> > + published_rel *table_info =
> > (published_rel *) malloc(sizeof(published_rel));
> > +
> > + table_info->relid = lfirst_oid(lc);
> > + table_info->pubid = pub_elem->oid;
> > + table_infos = lappend(table_infos, table_info);
> > + }
> > +
> > + tables = list_concat(tables, pub_elem_tables);
> >
> > Thoughts?
>
> I think we could only deduplicate published tables per publication to get all
> row filters and column lists for each published table later.
> I removed the temporary list 'tables' and modified the API of the function
> filter_partitions to handle published_rel type list.
>
> Attach the new patch set.
Thanks for the update patch.
One suggestion:
+/* Records association between publication and published table */
+typedef struct
+{
+ Oid relid; /* OID of published table */
+ Oid pubid; /* OID of publication
that publishes this
+ * table. */
+} published_rel;
+
+ /*
+ * Record the published table and the
corresponding publication so
+ * that we can get row filters and column list later.
+ *
+ * When a table is published by multiple
publications, to obtain
+ * all row filters and column list, the
structure related to this
+ * table will be recorded multiple times.
+ */
+ foreach(lc, pub_elem_tables)
+ {
+ published_rel *table_info =
(published_rel *) malloc(sizeof(published_rel));
+
+ table_info->relid = lfirst_oid(lc);
+ table_info->pubid = pub_elem->oid;
+ table_infos = lappend(table_infos, table_info);
+ }
In this format if there are n relations in publication we will store
pubid n times, in all tables publication there will many thousands of
tables. We could avoid storing the pubid for every relid, instead we
could represent it like below to avoid storing publication id for
each tables:
+/* Records association between publication and published tables */
+typedef struct
+{
+ List *relids, /* OIDs of the publisher tables */
+ Oid pubid; /* OID of publication that publishes this
+ * tables. */
+}published_rel;
Thoughts?
Regards,
Vignesh
^ permalink raw reply [nested|flat] 99+ messages in thread
* RE: Data is copied twice when specifying both child and parent table in publication
@ 2022-11-17 07:43 [email protected] <[email protected]>
parent: vignesh C <[email protected]>
0 siblings, 0 replies; 99+ messages in thread
From: [email protected] @ 2022-11-17 07:43 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: Peter Smith <[email protected]>; Takamichi Osumi (Fujitsu) <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Amit Langote <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Dilip Kumar <[email protected]>; Greg Nancarrow <[email protected]>
On Thurs, Nov 17, 2022 at 13:58 PM vignesh C <[email protected]> wrote:
> On Wed, 16 Nov 2022 at 14:28, [email protected]
> <[email protected]> wrote:
> >
> > On Mon, Nov 14, 2022 at 0:56 AM vignesh C <[email protected]> wrote:
> > > >
> > > > Attach new patches.
> > >
> >
> > Thanks for your comments.
> >
> > > Here we are having tables list to store the relids and table_infos
> > > list which stores pubid along with relid. Here tables list acts as a
> > > temporary list to get filter_partitions and then delete the
> > > published_rel from table_infos. Will it be possible to directly
> > > operate on table_infos list and remove the temporary tables list used.
> > > We might have to implement comparator, deduplication functions and
> > > change filter_partitions function to work directly on published_rel
> > > type list.
> > > + /
> > > + * Record the published table and the
> > > corresponding publication so
> > > + * that we can get row filters and column list later.
> > > + *
> > > + * When a table is published by multiple
> > > publications, to obtain
> > > + * all row filters and column list, the
> > > structure related to this
> > > + * table will be recorded multiple times.
> > > + */
> > > + foreach(lc, pub_elem_tables)
> > > + {
> > > + published_rel *table_info =
> > > (published_rel *) malloc(sizeof(published_rel));
> > > +
> > > + table_info->relid = lfirst_oid(lc);
> > > + table_info->pubid = pub_elem->oid;
> > > + table_infos = lappend(table_infos, table_info);
> > > + }
> > > +
> > > + tables = list_concat(tables, pub_elem_tables);
> > >
> > > Thoughts?
> >
> > I think we could only deduplicate published tables per publication to get all
> > row filters and column lists for each published table later.
> > I removed the temporary list 'tables' and modified the API of the function
> > filter_partitions to handle published_rel type list.
> >
> > Attach the new patch set.
>
> Thanks for the update patch.
Thanks for your comment.
> One suggestion:
> +/* Records association between publication and published table */
> +typedef struct
> +{
> + Oid relid; /* OID of published table */
> + Oid pubid; /* OID of publication
> that publishes this
> + * table. */
> +} published_rel;
> +
>
> + /*
> + * Record the published table and the
> corresponding publication so
> + * that we can get row filters and column list later.
> + *
> + * When a table is published by multiple
> publications, to obtain
> + * all row filters and column list, the
> structure related to this
> + * table will be recorded multiple times.
> + */
> + foreach(lc, pub_elem_tables)
> + {
> + published_rel *table_info =
> (published_rel *) malloc(sizeof(published_rel));
> +
> + table_info->relid = lfirst_oid(lc);
> + table_info->pubid = pub_elem->oid;
> + table_infos = lappend(table_infos, table_info);
> + }
>
> In this format if there are n relations in publication we will store
> pubid n times, in all tables publication there will many thousands of
> tables. We could avoid storing the pubid for every relid, instead we
> could represent it like below to avoid storing publication id for
> each tables:
>
> +/* Records association between publication and published tables */
> +typedef struct
> +{
> + List *relids, /* OIDs of the publisher tables */
> + Oid pubid; /* OID of publication that publishes this
> + * tables. */
> +}published_rel;
>
> Thoughts?
I think this complicates the function filter_partitions.
Because if we use such a node type, I think we need to concatenate 'relids'
list of each node of the 'table_infos' list in the function filter_partitions
to become a temporary list. Then filter this temporary list and process the
'table_infos' list according to the filtering result.
Regards,
Wang wei
^ permalink raw reply [nested|flat] 99+ messages in thread
end of thread, other threads:[~2022-11-17 07:43 UTC | newest]
Thread overview: 99+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-07-15 08:19 [PATCH 3/4] f!convert the other verbose to int, too Justin Pryzby <[email protected]>
2021-10-15 11:22 Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-10-16 06:30 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-10-18 02:57 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-18 06:00 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-18 09:02 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Langote <[email protected]>
2021-10-18 09:28 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-19 03:15 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-19 04:29 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-19 02:47 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-10-19 05:37 ` Re: Data is copied twice when specifying both child and parent table in publication Dilip Kumar <[email protected]>
2021-10-28 07:35 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-10-28 08:25 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Langote <[email protected]>
2021-11-03 10:13 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-11-04 04:13 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-11-04 06:53 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-11-04 08:10 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-11-05 03:20 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-11-11 06:52 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-11-11 08:14 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-11-12 04:11 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-11-12 04:27 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-11-16 01:56 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-11-29 08:51 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-12-01 11:15 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-12-01 23:21 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-12-02 02:33 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-12-02 02:48 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-12-02 04:10 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-12-02 04:50 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-12-02 08:54 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-12-03 05:54 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-12-07 12:23 ` Re: Data is copied twice when specifying both child and parent table in publication vignesh C <[email protected]>
2021-12-08 05:41 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-12-08 06:00 ` Re: Data is copied twice when specifying both child and parent table in publication vignesh C <[email protected]>
2021-12-08 06:08 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-12-09 10:51 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2022-04-19 07:04 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-04-19 08:53 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-04-21 03:05 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-04-21 09:41 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2022-04-24 06:16 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-04-25 01:23 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-04-28 01:22 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-09 01:51 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-11 02:32 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-12 01:47 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-13 02:02 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-13 05:58 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2022-05-13 09:41 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-13 14:57 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-18 08:39 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-17 13:02 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2022-05-18 08:37 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-05-18 08:51 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-07-01 09:46 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-07-14 04:45 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-07-21 10:07 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-07-28 09:17 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-08-04 11:26 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-08-05 09:06 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-08-09 07:14 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-08-30 07:43 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-09-19 06:51 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-09-20 06:17 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-09-26 02:31 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-09-26 04:44 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-09-27 08:44 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-09-28 08:35 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-10-05 03:17 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-10-17 05:49 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-10-21 08:52 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-11-11 05:43 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-11-13 16:55 ` Re: Data is copied twice when specifying both child and parent table in publication vignesh C <[email protected]>
2022-11-16 08:58 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-11-17 05:57 ` Re: Data is copied twice when specifying both child and parent table in publication vignesh C <[email protected]>
2022-11-17 07:43 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-10-21 09:01 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-11-11 05:44 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-11-03 23:34 ` Re: Data is copied twice when specifying both child and parent table in publication Ian Lawrence Barwick <[email protected]>
2022-11-08 04:12 ` RE: Data is copied twice when specifying both child and parent table in publication Takamichi Osumi (Fujitsu) <[email protected]>
2022-11-11 05:45 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-10-05 15:04 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-10-17 05:52 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2022-08-09 23:45 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2022-08-30 02:24 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-12-02 03:30 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Langote <[email protected]>
2021-10-22 02:01 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
2021-10-20 07:14 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-20 08:02 ` Re: Data is copied twice when specifying both child and parent table in publication Dilip Kumar <[email protected]>
2021-10-20 08:59 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-20 09:33 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-20 10:19 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-20 13:40 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-21 03:56 ` Re: Data is copied twice when specifying both child and parent table in publication Peter Smith <[email protected]>
2021-10-21 05:57 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-21 09:45 ` Re: Data is copied twice when specifying both child and parent table in publication Amit Kapila <[email protected]>
2021-10-20 09:00 ` Re: Data is copied twice when specifying both child and parent table in publication Greg Nancarrow <[email protected]>
2021-10-18 08:46 ` RE: Data is copied twice when specifying both child and parent table in publication [email protected] <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox